- addr = bp->start + (page << bp->log2_page_size) + slot * bp->buffer_size;
-
- return uword_to_pointer (addr, void *);
-}
-
-/* Make sure free list has at least given number of free buffers. */
-static uword
-vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl,
- uword min_free_buffers)
-{
- vlib_buffer_t *b;
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (fl->buffer_pool_index);
- int n;
- u32 *bi;
- u32 n_alloc = 0;
-
- /* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
-
- if (vec_len (bp->buffers) > 0)
- {
- int n_copy, n_left;
- clib_spinlock_lock (&bp->lock);
- n_copy = clib_min (vec_len (bp->buffers), n);
- n_left = vec_len (bp->buffers) - n_copy;
- vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
- CLIB_CACHE_LINE_BYTES);
- _vec_len (bp->buffers) = n_left;
- clib_spinlock_unlock (&bp->lock);
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
- return min_free_buffers;
- }
-
- /* Always allocate round number of buffers. */
- n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
-
- /* Always allocate new buffers in reasonably large sized chunks. */
- n = clib_max (n, fl->min_n_buffers_each_alloc);
-
- clib_spinlock_lock (&bp->lock);
- while (n_alloc < n)
- {
- if ((b = vlib_buffer_pool_get_buffer (bp)) == 0)
- goto done;
-
- n_alloc += 1;
-
- vec_add2_aligned (fl->buffers, bi, 1, CLIB_CACHE_LINE_BYTES);
- bi[0] = vlib_get_buffer_index (vm, b);
-
- if (CLIB_DEBUG > 0)
- vlib_buffer_set_known_state (bi[0], VLIB_BUFFER_KNOWN_FREE);
-
- memset (b, 0, sizeof (vlib_buffer_t));
- vlib_buffer_init_for_free_list (b, fl);
-
- if (fl->buffer_init_function)
- fl->buffer_init_function (vm, fl, bi, 1);
- }
-
-done:
- clib_spinlock_unlock (&bp->lock);
- fl->n_alloc += n_alloc;
- return n_alloc;
-}
-
-void *
-vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
-{
- vlib_buffer_main_t *bm = &buffer_main;
- void *rv = bm->buffer_free_callback;
-
- bm->buffer_free_callback = fp;
- return rv;
-}
-
-static_always_inline void
-recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
- vlib_buffer_t * b, u32 follow_buffer_next)
-{
- vlib_buffer_free_list_t *fl;
- vlib_buffer_free_list_index_t fi;
- fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
-
- /* The only current use of this callback:
- * multicast recycle */
- if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
- {
- int j;
-
- vlib_buffer_add_to_free_list (vm, fl, bi,
- (b->flags & VLIB_BUFFER_RECYCLE) == 0);
- for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
- {
- if (fl == vm->buffer_announce_list[j])
- goto already_announced;
- }
- vec_add1 (vm->buffer_announce_list, fl);
- already_announced:
- ;
- }
- else
- {
- if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- {
- u32 flags, next;
-
- do
- {
- vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
- flags = nb->flags;
- next = nb->next_buffer;
- if (nb->n_add_refs)
- nb->n_add_refs--;
- else
- {
- vlib_buffer_validate_alloc_free (vm, &bi, 1,
- VLIB_BUFFER_KNOWN_ALLOCATED);
- vlib_buffer_add_to_free_list (vm, fl, bi, 1);
- }
- bi = next;
- }
- while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
-
- }
- }
-}
-
-static_always_inline void
-vlib_buffer_free_inline (vlib_main_t * vm,
- u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
-{
- vlib_buffer_main_t *bm = &buffer_main;
- vlib_buffer_t *p, *b0, *b1, *b2, *b3;
- int i = 0;
- u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
- u32 follow_buffer_next);
-
- cb = bm->buffer_free_callback;
-
- if (PREDICT_FALSE (cb != 0))
- n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
-
- if (!n_buffers)
- return;
-
- while (i + 11 < n_buffers)
- {
- p = vlib_get_buffer (vm, buffers[i + 8]);
- vlib_prefetch_buffer_header (p, LOAD);
- p = vlib_get_buffer (vm, buffers[i + 9]);
- vlib_prefetch_buffer_header (p, LOAD);
- p = vlib_get_buffer (vm, buffers[i + 10]);
- vlib_prefetch_buffer_header (p, LOAD);
- p = vlib_get_buffer (vm, buffers[i + 11]);
- vlib_prefetch_buffer_header (p, LOAD);
-
- b0 = vlib_get_buffer (vm, buffers[i]);
- b1 = vlib_get_buffer (vm, buffers[i + 1]);
- b2 = vlib_get_buffer (vm, buffers[i + 2]);
- b3 = vlib_get_buffer (vm, buffers[i + 3]);
-
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
-
- recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
- recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next);
- recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next);
- recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next);
-
- i += 4;
- }
-
- while (i < n_buffers)
- {
- b0 = vlib_get_buffer (vm, buffers[i]);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
- i++;
- }
-
- if (vec_len (vm->buffer_announce_list))
- {
- vlib_buffer_free_list_t *fl;
- for (i = 0; i < vec_len (vm->buffer_announce_list); i++)