-
- /* Aligned copy. */
- {
- vlib_copy_unit_t *d, *s;
- uword n_copy;
-
- if (vec_len (free_list->aligned_buffers) <
- ((n_left / BUFFERS_PER_COPY) * BUFFERS_PER_COPY))
- abort ();
-
- n_copy = n_left / BUFFERS_PER_COPY;
- n_left = n_left % BUFFERS_PER_COPY;
-
- /* Remove buffers from aligned free list. */
- _vec_len (free_list->aligned_buffers) -= n_copy * BUFFERS_PER_COPY;
-
- s = (vlib_copy_unit_t *) vec_end (free_list->aligned_buffers);
- d = (vlib_copy_unit_t *) dst;
-
- /* Fast path loop. */
- while (n_copy >= 4)
- {
- d[0] = s[0];
- d[1] = s[1];
- d[2] = s[2];
- d[3] = s[3];
- n_copy -= 4;
- s += 4;
- d += 4;
- }
-
- while (n_copy >= 1)
- {
- d[0] = s[0];
- n_copy -= 1;
- s += 1;
- d += 1;
- }
-
- dst = (void *) d;
- }
-
- /* Unaligned copy. */
- ASSERT (n_unaligned_end == n_left);
- while (n_left > 0)
- {
- *dst++ = *u_src--;
- n_left--;
- u_len--;
- }
-
- if (!free_list->unaligned_buffers)
- ASSERT (u_len == 0);
- else
- _vec_len (free_list->unaligned_buffers) = u_len;
-
- /* Verify that buffers are known free. */
- vlib_buffer_validate_alloc_free (vm, alloc_buffers,
- n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
-
- return n_alloc_buffers;
-}
-
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-static u32
-vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-static u32
-vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
-always_inline void
-add_buffer_to_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * f,
- u32 buffer_index, u8 do_init)
-{
- vlib_buffer_t *b;
- b = vlib_get_buffer (vm, buffer_index);
- if (PREDICT_TRUE (do_init))
- vlib_buffer_init_for_free_list (b, f);
- vec_add1_aligned (f->aligned_buffers, buffer_index,
- sizeof (vlib_copy_unit_t));
-}
-
-always_inline vlib_buffer_free_list_t *
-buffer_get_free_list (vlib_main_t * vm, vlib_buffer_t * b, u32 * index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- u32 i;
-
- *index = i = b->free_list_index;
- return pool_elt_at_index (bm->buffer_free_list_pool, i);
-}
-
-void *
-vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- void *rv = bm->buffer_free_callback;
-
- bm->buffer_free_callback = fp;
- return rv;