#endif
/* Make sure free list has at least given number of free buffers. */
-static uword
-fill_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl, uword min_free_buffers)
+uword
+CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm,
+ vlib_buffer_free_list_t * fl,
+ uword min_free_buffers)
{
dpdk_main_t *dm = &dpdk_main;
vlib_buffer_t *b0, *b1, *b2, *b3;
return n;
}
-static u32
-alloc_from_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- u32 * alloc_buffers, u32 n_alloc_buffers)
-{
- u32 *dst, *src;
- uword len, n_filled;
-
- dst = alloc_buffers;
-
- n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
- if (n_filled == 0)
- return 0;
-
- len = vec_len (free_list->buffers);
- ASSERT (len >= n_alloc_buffers);
-
- src = free_list->buffers + len - n_alloc_buffers;
- clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
- _vec_len (free_list->buffers) -= n_alloc_buffers;
-
- return n_alloc_buffers;
-}
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
- u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers,
- u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
static_always_inline void
dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
{
/* *INDENT-OFF* */
VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
- .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
- .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
+ .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
.vlib_buffer_free_cb = &dpdk_buffer_free,
.vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
.vlib_packet_template_init_cb = &dpdk_packet_template_init,
/* *INDENT-ON* */
#if __x86_64__
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx512;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2;
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512;
dpdk_input_multiarch_select (void)
{
vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
- if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+ if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx512;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512;
}
- else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+ else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx2;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2;
}
/* When dubugging validate that given buffers are either known allocated
or known free. */
-static void
+void
vlib_buffer_validate_alloc_free (vlib_main_t * vm,
u32 * buffers,
uword n_buffers,
if (CLIB_DEBUG == 0)
return;
+ if (vlib_buffer_callbacks)
+ return;
+
is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
b = buffers;
for (i = 0; i < n_buffers; i++)
/* Make sure free list has at least given number of free buffers. */
static uword
-fill_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl, uword min_free_buffers)
+vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
+ vlib_buffer_free_list_t * fl,
+ uword min_free_buffers)
{
vlib_buffer_t *buffers, *b;
vlib_buffer_free_list_t *mfl;
return n_alloc;
}
-static u32
-alloc_from_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- u32 * alloc_buffers, u32 n_alloc_buffers)
-{
- u32 *dst, *src;
- uword len;
- uword n_filled;
-
- dst = alloc_buffers;
-
- n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
- if (n_filled == 0)
- return 0;
-
- len = vec_len (free_list->buffers);
- ASSERT (len >= n_alloc_buffers);
-
- src = free_list->buffers + len - n_alloc_buffers;
- clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
- _vec_len (free_list->buffers) -= n_alloc_buffers;
-
- /* Verify that buffers are known free. */
- vlib_buffer_validate_alloc_free (vm, alloc_buffers,
- n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
-
- return n_alloc_buffers;
-}
-
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-static u32
-vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-static u32
-vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
void *
vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
{
return 0;
}
- bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal;
- bm->cb.vlib_buffer_alloc_from_free_list_cb =
- &vlib_buffer_alloc_from_free_list_internal;
+ bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal;
bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal;
bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
bm->cb.vlib_buffer_delete_free_list_cb =
uword buffer_init_function_opaque;
} __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
-typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers,
- u32 n_buffers);
-typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers,
- u32 free_list_index);
+typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm,
+ vlib_buffer_free_list_t * fl,
+ uword min_free_buffers);
typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers,
u32 n_buffers);
typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
typedef struct
{
- vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb;
- vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb;
+ vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb;
vlib_buffer_free_cb_t *vlib_buffer_free_cb;
vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb;
void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
VLIB_BUFFER_KNOWN_ALLOCATED,
} vlib_buffer_known_state_t;
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+ uword n_buffers,
+ vlib_buffer_known_state_t
+ expected_state);
+
always_inline vlib_buffer_known_state_t
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-/** \brief Allocate buffers into supplied array
-
- @param vm - (vlib_main_t *) vlib main data structure pointer
- @param buffers - (u32 * ) buffer index array
- @param n_buffers - (u32) number of buffers requested
- @return - (u32) number of buffers actually allocated, may be
- less than the number requested or zero
-*/
-always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_alloc_cb);
-
- return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
-}
-
always_inline u32
vlib_buffer_round_size (u32 size)
{
u32 n_buffers, u32 free_list_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_free_list_t *fl;
+ u32 *src;
+ uword len;
+
+ ASSERT (bm->cb.vlib_buffer_fill_free_list_cb);
+
+ fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
+
+ len = vec_len (fl->buffers);
+
+ if (PREDICT_FALSE (len < n_buffers))
+ {
+ bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
+ len = vec_len (fl->buffers);
- ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+ /* even if fill free list didn't manage to refill free list
+ we should give what we have */
+ n_buffers = clib_min (len, n_buffers);
- return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
- free_list_index);
+ /* following code is intentionaly duplicated to allow compiler
+ to optimize fast path when n_buffers is constant value */
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
+
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+
+ return n_buffers;
+ }
+
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
+
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+
+ return n_buffers;
+}
+
+/** \brief Allocate buffers into supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u32) number of buffers requested
+ @return - (u32) number of buffers actually allocated, may be
+ less than the number requested or zero
+*/
+always_inline u32
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+ return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
}
/** \brief Free buffers
void *
mc_get_vlib_buffer (vlib_main_t * vm, u32 n_bytes, u32 * bi_return)
{
- u32 n_alloc, bi;
+ u32 n_alloc, bi = 0;
vlib_buffer_t *b;
n_alloc = vlib_buffer_alloc (vm, &bi, 1);
dns_cache_entry_t * ep, vlib_buffer_t * b0)
{
vlib_main_t *vm = dm->vlib_main;
- u32 bi;
+ u32 bi = 0;
fib_prefix_t prefix;
fib_node_index_t fei;
u32 sw_if_index, fib_index;