- known = vlib_buffer_is_known (bi);
- if (known != expected_state)
- {
- ASSERT (0);
- vlib_panic_with_msg
- (vm, "%s %U buffer 0x%x",
- is_free ? "freeing" : "allocating",
- format_vlib_buffer_known_state, known, bi);
- }
-
- vlib_buffer_set_known_state
- (bi, is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED);
- }
-}
-
-/* Add buffer free list. */
-static vlib_buffer_free_list_index_t
-vlib_buffer_create_free_list_helper (vlib_main_t * vm,
- u32 n_data_bytes,
- u32 is_public, u32 is_default, u8 * name)
-{
- vlib_buffer_main_t *bm = &buffer_main;
- vlib_buffer_free_list_t *f;
- int i;
-
- ASSERT (vlib_get_thread_index () == 0);
-
- if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0)
- {
- vlib_buffer_free_list_index_t default_free_free_list_index;
-
- /* *INDENT-OFF* */
- default_free_free_list_index =
- vlib_buffer_create_free_list_helper
- (vm,
- /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
- /* is_public */ 1,
- /* is_default */ 1,
- (u8 *) "default");
- /* *INDENT-ON* */
- ASSERT (default_free_free_list_index ==
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
-
- if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
- return default_free_free_list_index;
- }
-
- pool_get_aligned (vm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES);
-
- clib_memset (f, 0, sizeof (f[0]));
- f->index = f - vm->buffer_free_list_pool;
- f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
- f->min_n_buffers_each_alloc = VLIB_FRAME_SIZE;
- f->buffer_pool_index = 0;
- f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name);
-
- /* Setup free buffer template. */
- vlib_buffer_set_free_list_index (&f->buffer_init_template, f->index);
- f->buffer_init_template.n_add_refs = 0;
-
- if (is_public)
- {
- uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes);
- if (!p)
- hash_set (bm->free_list_by_size, f->n_data_bytes, f->index);
- }
-
- for (i = 1; i < vec_len (vlib_mains); i++)
- {
- vlib_main_t *wvm = vlib_mains[i];
- vlib_buffer_free_list_t *wf;
- pool_get_aligned (wvm->buffer_free_list_pool,
- wf, CLIB_CACHE_LINE_BYTES);
- ASSERT (f - vm->buffer_free_list_pool ==
- wf - wvm->buffer_free_list_pool);
- wf[0] = f[0];
- wf->buffers = 0;
- wf->n_alloc = 0;
- }
-
- return f->index;
-}
-
-vlib_buffer_free_list_index_t
-vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
- char *fmt, ...)
-{
- va_list va;
- u8 *name;
-
- va_start (va, fmt);
- name = va_format (0, fmt, &va);
- va_end (va);
-
- return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
- /* is_public */ 0,
- /* is_default */ 0,
- name);
-}
-
-static void
-del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
-{
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index);
-
- vec_add_aligned (bp->buffers, f->buffers, vec_len (f->buffers),
- CLIB_CACHE_LINE_BYTES);
- vec_free (f->name);
- vec_free (f->buffers);
-
- /* Poison it. */
- clib_memset (f, 0xab, sizeof (f[0]));
-}
-
-/* Add buffer free list. */
-void
-vlib_buffer_delete_free_list_internal (vlib_main_t * vm,
- vlib_buffer_free_list_index_t index)
-{
- vlib_buffer_free_list_t *f;
- int i;
-
- ASSERT (vlib_get_thread_index () == 0);
-
- f = vlib_buffer_get_free_list (vm, index);
-
- ASSERT (vec_len (f->buffers) == f->n_alloc);
-
- del_free_list (vm, f);
-
- pool_put (vm->buffer_free_list_pool, f);
-
- for (i = 1; i < vec_len (vlib_mains); i++)
- {
- vlib_main_t *wvm = vlib_mains[i];
- f = vlib_buffer_get_free_list (vlib_mains[i], index);
- del_free_list (wvm, f);
- pool_put (wvm->buffer_free_list_pool, f);
- }
-}
-
-static_always_inline void *
-vlib_buffer_pool_get_buffer (vlib_main_t * vm, vlib_buffer_pool_t * bp)
-{
- return vlib_physmem_alloc_from_map (vm, bp->physmem_map_index,
- bp->buffer_size, CLIB_CACHE_LINE_BYTES);
-}
-
-/* Make sure free list has at least given number of free buffers. */
-static uword
-vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl,
- uword min_free_buffers)
-{
- vlib_buffer_t *b;
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (fl->buffer_pool_index);
- int n;
- u32 *bi;
- u32 n_alloc = 0;
-
- /* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
-
- if (vec_len (bp->buffers) > 0)
- {
- int n_copy, n_left;
- clib_spinlock_lock (&bp->lock);
- n_copy = clib_min (vec_len (bp->buffers), n);
- n_left = vec_len (bp->buffers) - n_copy;
- vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
- CLIB_CACHE_LINE_BYTES);
- _vec_len (bp->buffers) = n_left;
- clib_spinlock_unlock (&bp->lock);
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
- }
-
- /* Always allocate round number of buffers. */
- n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
-
- /* Always allocate new buffers in reasonably large sized chunks. */
- n = clib_max (n, fl->min_n_buffers_each_alloc);
-
- clib_spinlock_lock (&bp->lock);
- while (n_alloc < n)
- {
- if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0)
- goto done;