- /* Setup free buffer template. */
- vlib_buffer_set_free_list_index (&f->buffer_init_template, f->index);
- f->buffer_init_template.n_add_refs = 0;
-
- if (is_public)
- {
- uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes);
- if (!p)
- hash_set (bm->free_list_by_size, f->n_data_bytes, f->index);
- }
-
- for (i = 1; i < vec_len (vlib_mains); i++)
- {
- vlib_main_t *wvm = vlib_mains[i];
- vlib_buffer_free_list_t *wf;
- pool_get_aligned (wvm->buffer_free_list_pool,
- wf, CLIB_CACHE_LINE_BYTES);
- ASSERT (f - vm->buffer_free_list_pool ==
- wf - wvm->buffer_free_list_pool);
- wf[0] = f[0];
- wf->buffers = 0;
- wf->n_alloc = 0;
- }
-
- return f->index;
-}
-
-vlib_buffer_free_list_index_t
-vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
- char *fmt, ...)
-{
- va_list va;
- u8 *name;
-
- va_start (va, fmt);
- name = va_format (0, fmt, &va);
- va_end (va);
-
- return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
- /* is_public */ 0,
- /* is_default */ 0,
- name);
-}
-
-static void
-del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
-{
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index);
-
- vec_add_aligned (bp->buffers, f->buffers, vec_len (f->buffers),
- CLIB_CACHE_LINE_BYTES);
- vec_free (f->name);
- vec_free (f->buffers);
-
- /* Poison it. */
- clib_memset (f, 0xab, sizeof (f[0]));
-}
-
-/* Add buffer free list. */
-void
-vlib_buffer_delete_free_list_internal (vlib_main_t * vm,
- vlib_buffer_free_list_index_t index)
-{
- vlib_buffer_free_list_t *f;
- int i;
-
- ASSERT (vlib_get_thread_index () == 0);
-
- f = vlib_buffer_get_free_list (vm, index);
-
- ASSERT (vec_len (f->buffers) == f->n_alloc);
-
- del_free_list (vm, f);
-
- pool_put (vm->buffer_free_list_pool, f);
-
- for (i = 1; i < vec_len (vlib_mains); i++)
- {
- vlib_main_t *wvm = vlib_mains[i];
- f = vlib_buffer_get_free_list (vlib_mains[i], index);
- del_free_list (wvm, f);
- pool_put (wvm->buffer_free_list_pool, f);
- }
-}
-
-static_always_inline void *
-vlib_buffer_pool_get_buffer (vlib_main_t * vm, vlib_buffer_pool_t * bp)
-{
- return vlib_physmem_alloc_from_map (vm, bp->physmem_map_index,
- bp->buffer_size, CLIB_CACHE_LINE_BYTES);
-}
-
-/* Make sure free list has at least given number of free buffers. */
-static uword
-vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl,
- uword min_free_buffers)
-{
- vlib_buffer_t *b;
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (fl->buffer_pool_index);
- int n;
- u32 *bi;
- u32 n_alloc = 0;
-
- /* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
-
- if (vec_len (bp->buffers) > 0)
- {
- int n_copy, n_left;
- clib_spinlock_lock (&bp->lock);
- n_copy = clib_min (vec_len (bp->buffers), n);
- n_left = vec_len (bp->buffers) - n_copy;
- vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
- CLIB_CACHE_LINE_BYTES);
- _vec_len (bp->buffers) = n_left;
- clib_spinlock_unlock (&bp->lock);
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
- }
-
- /* Always allocate round number of buffers. */
- n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
-
- /* Always allocate new buffers in reasonably large sized chunks. */
- n = clib_max (n, fl->min_n_buffers_each_alloc);
-
- clib_spinlock_lock (&bp->lock);
- while (n_alloc < n)
- {
- if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0)
- goto done;
-
- n_alloc += 1;
-
- vec_add2_aligned (fl->buffers, bi, 1, CLIB_CACHE_LINE_BYTES);
- bi[0] = vlib_get_buffer_index (vm, b);
-
- if (CLIB_DEBUG > 0)
- vlib_buffer_set_known_state (bi[0], VLIB_BUFFER_KNOWN_FREE);
-
- clib_memset (b, 0, sizeof (vlib_buffer_t));
- vlib_buffer_init_for_free_list (b, fl);
-
- if (fl->buffer_init_function)
- fl->buffer_init_function (vm, fl, bi, 1);
- }
-
-done:
- clib_spinlock_unlock (&bp->lock);
- fl->n_alloc += n_alloc;
- return n_alloc;
-}
-
-void *
-vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
-{
- vlib_buffer_main_t *bm = &buffer_main;
- void *rv = bm->buffer_free_callback;
-
- bm->buffer_free_callback = fp;
- return rv;
-}
-
-static_always_inline void
-recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
- vlib_buffer_t * b, u32 follow_buffer_next)
-{
- vlib_buffer_free_list_t *fl;
- vlib_buffer_free_list_index_t fi;
- u32 flags, next;
-
- fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
-
- do
- {
- vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
- flags = nb->flags;
- next = nb->next_buffer;
- if (nb->n_add_refs)
- nb->n_add_refs--;
- else