- if (fl->buffer_init_function)
- fl->buffer_init_function (vm, fl, bi, n_this_chunk);
- }
- return n_alloc;
-}
-
-static u32
-alloc_from_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- u32 * alloc_buffers, u32 n_alloc_buffers)
-{
- u32 *dst, *src;
- uword len;
- uword n_filled;
-
- dst = alloc_buffers;
-
- /* wait with buffer memory allocation as long as possible
- in case external buffer manager takes over */
- if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0))
- unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
-
- n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
- if (n_filled == 0)
- return 0;
-
- len = vec_len (free_list->buffers);
- ASSERT (len >= n_alloc_buffers);
-
- src = free_list->buffers + len - n_alloc_buffers;
- clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
- _vec_len (free_list->buffers) -= n_alloc_buffers;
-
- /* Verify that buffers are known free. */
- vlib_buffer_validate_alloc_free (vm, alloc_buffers,
- n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
-
- return n_alloc_buffers;
-}
-
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-static u32
-vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-static u32
-vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
-void *
-vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- void *rv = bm->buffer_free_callback;
-
- bm->buffer_free_callback = fp;
- return rv;
-}
-
-static_always_inline void
-vlib_buffer_free_inline (vlib_main_t * vm,
- u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *fl;
- u32 fi;
- int i;
- u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
- u32 follow_buffer_next);
-
- cb = bm->buffer_free_callback;
-
- if (PREDICT_FALSE (cb != 0))
- n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
-
- if (!n_buffers)
- return;
-
- for (i = 0; i < n_buffers; i++)
- {
- vlib_buffer_t *b;
- u32 bi = buffers[i];
-
- b = vlib_get_buffer (vm, bi);
-
- fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
-
- /* The only current use of this callback: multicast recycle */
- if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
- {
- int j;
-
- vlib_buffer_add_to_free_list
- (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-
- for (j = 0; j < vec_len (bm->announce_list); j++)
- {
- if (fl == bm->announce_list[j])
- goto already_announced;
- }
- vec_add1 (bm->announce_list, fl);
- already_announced:
- ;
- }
- else
- {
- if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- {
- u32 flags, next;
-
- do
- {
- vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
- flags = nb->flags;
- next = nb->next_buffer;
- if (nb->n_add_refs)
- nb->n_add_refs--;
- else
- {
- vlib_buffer_validate_alloc_free (vm, &bi, 1,
- VLIB_BUFFER_KNOWN_ALLOCATED);
- vlib_buffer_add_to_free_list (vm, fl, bi, 1);
- }
- bi = next;
- }
- while (follow_buffer_next
- && (flags & VLIB_BUFFER_NEXT_PRESENT));
-
- }
- }
- }
- if (vec_len (bm->announce_list))
- {
- vlib_buffer_free_list_t *fl;
- for (i = 0; i < vec_len (bm->announce_list); i++)
- {
- fl = bm->announce_list[i];
- fl->buffers_added_to_freelist_function (vm, fl);
- }
- _vec_len (bm->announce_list) = 0;
- }
-}
-
-static void
-vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
- vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
- 1);
-}
-
-static void
-vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers,
- u32 n_buffers)
-{
- vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
- 0);
-}
-
-/* Copy template packet data into buffers as they are allocated. */
-static void __attribute__ ((unused))
-vlib_packet_template_buffer_init (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl,
- u32 * buffers, u32 n_buffers)
-{
- vlib_packet_template_t *t =
- uword_to_pointer (fl->buffer_init_function_opaque,
- vlib_packet_template_t *);
- uword i;
-
- for (i = 0; i < n_buffers; i++)
- {
- vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
- ASSERT (b->current_length == vec_len (t->packet_data));
- clib_memcpy (vlib_buffer_get_current (b), t->packet_data,
- b->current_length);