}
}
-#define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32))
-
-/* Make sure we have at least given number of unaligned buffers. */
-void
-vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- uword n_unaligned_buffers)
-{
- word la = vec_len (free_list->aligned_buffers);
- word lu = vec_len (free_list->unaligned_buffers);
-
- /* Aligned come in aligned copy-sized chunks. */
- ASSERT (la % BUFFERS_PER_COPY == 0);
-
- ASSERT (la >= n_unaligned_buffers);
-
- while (lu < n_unaligned_buffers)
- {
- /* Copy 4 buffers from end of aligned vector to unaligned vector. */
- vec_add (free_list->unaligned_buffers,
- free_list->aligned_buffers + la - BUFFERS_PER_COPY,
- BUFFERS_PER_COPY);
- la -= BUFFERS_PER_COPY;
- lu += BUFFERS_PER_COPY;
- }
- _vec_len (free_list->aligned_buffers) = la;
-}
-
-/* After free aligned buffers may not contain even sized chunks. */
-void
-vlib_buffer_free_list_trim_aligned (vlib_buffer_free_list_t * f)
-{
- uword l, n_trim;
-
- /* Add unaligned to aligned before trim. */
- l = vec_len (f->unaligned_buffers);
- if (l > 0)
- {
- vec_add_aligned (f->aligned_buffers, f->unaligned_buffers, l,
- /* align */ sizeof (vlib_copy_unit_t));
-
- _vec_len (f->unaligned_buffers) = 0;
- }
-
- /* Remove unaligned buffers from end of aligned vector and save for next trim. */
- l = vec_len (f->aligned_buffers);
- n_trim = l % BUFFERS_PER_COPY;
- if (n_trim)
- {
- /* Trim aligned -> unaligned. */
- vec_add (f->unaligned_buffers, f->aligned_buffers + l - n_trim, n_trim);
-
- /* Remove from aligned. */
- _vec_len (f->aligned_buffers) = l - n_trim;
- }
-}
-
void
vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
vlib_buffer_free_list_t * src)
uword l;
u32 *d;
- vlib_buffer_free_list_trim_aligned (src);
- vlib_buffer_free_list_trim_aligned (dst);
-
- l = vec_len (src->aligned_buffers);
- if (l > 0)
- {
- vec_add2_aligned (dst->aligned_buffers, d, l,
- /* align */ sizeof (vlib_copy_unit_t));
- clib_memcpy (d, src->aligned_buffers, l * sizeof (d[0]));
- vec_free (src->aligned_buffers);
- }
-
- l = vec_len (src->unaligned_buffers);
+ l = vec_len (src->buffers);
if (l > 0)
{
- vec_add (dst->unaligned_buffers, src->unaligned_buffers, l);
- vec_free (src->unaligned_buffers);
+ vec_add2_aligned (dst->buffers, d, l, CLIB_CACHE_LINE_BYTES);
+ clib_memcpy (d, src->buffers, l * sizeof (d[0]));
+ vec_free (src->buffers);
}
}
ASSERT (f - bm->buffer_free_list_pool ==
wf - wbm->buffer_free_list_pool);
wf[0] = f[0];
- wf->aligned_buffers = 0;
- wf->unaligned_buffers = 0;
+ wf->buffers = 0;
wf->n_alloc = 0;
}
vm->os_physmem_free (f->buffer_memory_allocated[i]);
vec_free (f->name);
vec_free (f->buffer_memory_allocated);
- vec_free (f->unaligned_buffers);
- vec_free (f->aligned_buffers);
+ vec_free (f->buffers);
}
/* Add buffer free list. */
f = vlib_buffer_get_free_list (vm, free_list_index);
- ASSERT (vec_len (f->unaligned_buffers) + vec_len (f->aligned_buffers) ==
- f->n_alloc);
+ ASSERT (vec_len (f->buffers) == f->n_alloc);
merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
if (merge_index != ~0 && merge_index != free_list_index)
{
u32 *bi;
u32 n_remaining, n_alloc, n_this_chunk;
- vlib_buffer_free_list_trim_aligned (fl);
-
/* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->aligned_buffers);
+ n = min_free_buffers - vec_len (fl->buffers);
if (n <= 0)
return min_free_buffers;
/* Always allocate round number of buffers. */
- n = round_pow2 (n, BUFFERS_PER_COPY);
+ n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
/* Always allocate new buffers in reasonably large sized chunks. */
n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
n_remaining -= n_this_chunk;
b = buffers;
- vec_add2_aligned (fl->aligned_buffers, bi, n_this_chunk,
- sizeof (vlib_copy_unit_t));
+ vec_add2_aligned (fl->buffers, bi, n_this_chunk, CLIB_CACHE_LINE_BYTES);
for (i = 0; i < n_this_chunk; i++)
{
bi[i] = vlib_get_buffer_index (vm, b);
return n_alloc;
}
-always_inline uword
-copy_alignment (u32 * x)
-{
- return (pointer_to_uword (x) / sizeof (x[0])) % BUFFERS_PER_COPY;
-}
-
-
static u32
alloc_from_free_list (vlib_main_t * vm,
vlib_buffer_free_list_t * free_list,
u32 * alloc_buffers, u32 n_alloc_buffers)
{
- u32 *dst, *u_src;
- uword u_len, n_left;
- uword n_unaligned_start, n_unaligned_end, n_filled;
+ u32 *dst, *src;
+ uword len;
+ uword n_filled;
- n_left = n_alloc_buffers;
dst = alloc_buffers;
- n_unaligned_start = ((BUFFERS_PER_COPY - copy_alignment (dst))
- & (BUFFERS_PER_COPY - 1));
n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
if (n_filled == 0)
return 0;
- n_left = n_filled < n_left ? n_filled : n_left;
- n_alloc_buffers = n_left;
-
- if (n_unaligned_start >= n_left)
- {
- n_unaligned_start = n_left;
- n_unaligned_end = 0;
- }
- else
- n_unaligned_end = copy_alignment (dst + n_alloc_buffers);
-
- vlib_buffer_free_list_fill_unaligned (vm, free_list,
- n_unaligned_start + n_unaligned_end);
-
- u_len = vec_len (free_list->unaligned_buffers);
- u_src = free_list->unaligned_buffers + u_len - 1;
+ len = vec_len (free_list->buffers);
+ ASSERT (len >= n_alloc_buffers);
- if (n_unaligned_start)
- {
- uword n_copy = n_unaligned_start;
- if (n_copy > n_left)
- n_copy = n_left;
- n_left -= n_copy;
-
- while (n_copy > 0)
- {
- *dst++ = *u_src--;
- n_copy--;
- u_len--;
- }
-
- /* Now dst should be aligned. */
- if (n_left > 0)
- ASSERT (pointer_to_uword (dst) % sizeof (vlib_copy_unit_t) == 0);
- }
-
- /* Aligned copy. */
- {
- vlib_copy_unit_t *d, *s;
- uword n_copy;
-
- if (vec_len (free_list->aligned_buffers) <
- ((n_left / BUFFERS_PER_COPY) * BUFFERS_PER_COPY))
- abort ();
-
- n_copy = n_left / BUFFERS_PER_COPY;
- n_left = n_left % BUFFERS_PER_COPY;
-
- /* Remove buffers from aligned free list. */
- _vec_len (free_list->aligned_buffers) -= n_copy * BUFFERS_PER_COPY;
-
- s = (vlib_copy_unit_t *) vec_end (free_list->aligned_buffers);
- d = (vlib_copy_unit_t *) dst;
-
- /* Fast path loop. */
- while (n_copy >= 4)
- {
- d[0] = s[0];
- d[1] = s[1];
- d[2] = s[2];
- d[3] = s[3];
- n_copy -= 4;
- s += 4;
- d += 4;
- }
-
- while (n_copy >= 1)
- {
- d[0] = s[0];
- n_copy -= 1;
- s += 1;
- d += 1;
- }
-
- dst = (void *) d;
- }
-
- /* Unaligned copy. */
- ASSERT (n_unaligned_end == n_left);
- while (n_left > 0)
- {
- *dst++ = *u_src--;
- n_left--;
- u_len--;
- }
+ src = free_list->buffers + len - n_alloc_buffers;
+ clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
- if (!free_list->unaligned_buffers)
- ASSERT (u_len == 0);
- else
- _vec_len (free_list->unaligned_buffers) = u_len;
+ _vec_len (free_list->buffers) -= n_alloc_buffers;
/* Verify that buffers are known free. */
vlib_buffer_validate_alloc_free (vm, alloc_buffers,
vlib_buffer_validate_alloc_free (vm, b,
n_left, VLIB_BUFFER_KNOWN_ALLOCATED);
- vec_add2_aligned (fl->aligned_buffers, f, n_left,
- /* align */ sizeof (vlib_copy_unit_t));
+ vec_add2_aligned (fl->buffers, f, n_left, CLIB_CACHE_LINE_BYTES);
n = next_to_free[i_next_to_free];
while (n_left >= 4)
f -= 2;
n -= free_next0 + free_next1;
- _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
+ _vec_len (fl->buffers) = f - fl->buffers;
fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
fl1 = pool_elt_at_index (bm->buffer_free_list_pool, fi1);
fl = pool_elt_at_index (bm->buffer_free_list_pool, fi);
}
- vec_add2_aligned (fl->aligned_buffers, f, n_left,
- /* align */ sizeof (vlib_copy_unit_t));
+ vec_add2_aligned (fl->buffers, f, n_left, CLIB_CACHE_LINE_BYTES);
}
while (n_left >= 1)
f -= 1;
n -= free_next0;
- _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
+ _vec_len (fl->buffers) = f - fl->buffers;
fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
fi = fi0;
fl = pool_elt_at_index (bm->buffer_free_list_pool, fi);
- vec_add2_aligned (fl->aligned_buffers, f, n_left,
- /* align */ sizeof (vlib_copy_unit_t));
+ vec_add2_aligned (fl->buffers, f, n_left, CLIB_CACHE_LINE_BYTES);
}
if (follow_buffer_next && ((n_left = n - next_to_free[i_next_to_free]) > 0))
goto again;
}
- _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
+ _vec_len (fl->buffers) = f - fl->buffers;
if (vec_len (announce_list))
{
"#Alloc", "#Free");
size = sizeof (vlib_buffer_t) + f->n_data_bytes;
- n_free = vec_len (f->aligned_buffers) + vec_len (f->unaligned_buffers);
+ n_free = vec_len (f->buffers);
bytes_alloc = size * f->n_alloc;
bytes_free = size * n_free;
#define VLIB_BUFFER_DATA_SIZE (2048)
#define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
-#if defined (CLIB_HAVE_VEC128) || defined (__aarch64__)
-typedef u8x16 vlib_copy_unit_t;
-#else
-typedef u64 vlib_copy_unit_t;
-#endif
-
/** \file
vlib buffer structure definition and a few select
access methods. This structure and the buffer allocation
/* Total number of buffers allocated from this free list. */
u32 n_alloc;
- /* Vector of free buffers. Each element is a byte offset into I/O heap.
- Aligned vectors always has naturally aligned vlib_copy_unit_t sized chunks
- of buffer indices. Unaligned vector has any left over. This is meant to
- speed up copy routines. */
- u32 *aligned_buffers, *unaligned_buffers;
+ /* Vector of free buffers. Each element is a byte offset into I/O heap. */
+ u32 *buffers;
/* Memory chunks allocated for this free list
recorded here so they can be freed when free list
u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
char *fmt, ...);
-
-/* After free aligned buffers may not contain even sized chunks. */
-void vlib_buffer_free_list_trim_aligned (vlib_buffer_free_list_t * f);
-
/* Merge two free lists */
void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
vlib_buffer_free_list_t * src);
return n;
}
-typedef union
-{
- vlib_buffer_t b;
- vlib_copy_unit_t i[sizeof (vlib_buffer_t) / sizeof (vlib_copy_unit_t)];
-}
-vlib_buffer_union_t;
-
/* Set a buffer quickly into "uninitialized" state. We want this to
be extremely cheap and arrange for all fields that need to be
initialized to be in the first 128 bits of the buffer. */
always_inline void
-vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
+vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst = (vlib_buffer_union_t *) _dst;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure vlib_buffer_t is cacheline aligned and sized */
ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
/* Make sure buffer template is sane. */
ASSERT (fl->index == fl->buffer_init_template.free_list_index);
- /* Copy template from src->current_data thru src->free_list_index */
- dst->i[0] = src->i[0];
- if (1 * sizeof (dst->i[0]) < 16)
- dst->i[1] = src->i[1];
- if (2 * sizeof (dst->i[0]) < 16)
- dst->i[2] = src->i[2];
-
/* Make sure it really worked. */
-#define _(f) ASSERT (dst->b.f == src->b.f)
+#define _(f) dst->f = src->f
_(current_data);
_(current_length);
_(flags);
_(free_list_index);
#undef _
- ASSERT (dst->b.total_length_not_including_first_buffer == 0);
+ ASSERT (dst->total_length_not_including_first_buffer == 0);
}
always_inline void
b = vlib_get_buffer (vm, buffer_index);
if (PREDICT_TRUE (do_init))
vlib_buffer_init_for_free_list (b, f);
- vec_add1_aligned (f->aligned_buffers, buffer_index,
- sizeof (vlib_copy_unit_t));
+ vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
}
always_inline void
-vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
- vlib_buffer_t * _dst1,
+vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0,
+ vlib_buffer_t * dst1,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst0 = (vlib_buffer_union_t *) _dst0;
- vlib_buffer_union_t *dst1 = (vlib_buffer_union_t *) _dst1;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
ASSERT (fl->index == fl->buffer_init_template.free_list_index);
- /* Copy template from src->current_data thru src->free_list_index */
- dst0->i[0] = dst1->i[0] = src->i[0];
- if (1 * sizeof (dst0->i[0]) < 16)
- dst0->i[1] = dst1->i[1] = src->i[1];
- if (2 * sizeof (dst0->i[0]) < 16)
- dst0->i[2] = dst1->i[2] = src->i[2];
-
/* Make sure it really worked. */
-#define _(f) ASSERT (dst0->b.f == src->b.f && dst1->b.f == src->b.f)
+#define _(f) dst0->f = src->f; dst1->f = src->f
_(current_data);
_(current_length);
_(flags);
_(free_list_index);
#undef _
- ASSERT (dst0->b.total_length_not_including_first_buffer == 0);
- ASSERT (dst1->b.total_length_not_including_first_buffer == 0);
+ ASSERT (dst0->total_length_not_including_first_buffer == 0);
+ ASSERT (dst1->total_length_not_including_first_buffer == 0);
}
#if CLIB_DEBUG > 0
== fl_clone - bm_clone->buffer_free_list_pool);
fl_clone[0] = fl_orig[0];
- fl_clone->aligned_buffers = 0;
- fl_clone->unaligned_buffers = 0;
+ fl_clone->buffers = 0;
fl_clone->n_alloc = 0;
}));
/* *INDENT-ON* */
STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
-#define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32))
-
static void
del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
{
struct rte_mbuf *mb;
vlib_buffer_t *b;
- for (i = 0; i < vec_len (f->unaligned_buffers); i++)
+ for (i = 0; i < vec_len (f->buffers); i++)
{
- b = vlib_get_buffer (vm, f->unaligned_buffers[i]);
- mb = rte_mbuf_from_vlib_buffer (b);
- ASSERT (rte_mbuf_refcnt_read (mb) == 1);
- rte_pktmbuf_free (mb);
- }
- for (i = 0; i < vec_len (f->aligned_buffers); i++)
- {
- b = vlib_get_buffer (vm, f->aligned_buffers[i]);
+ b = vlib_get_buffer (vm, f->buffers[i]);
mb = rte_mbuf_from_vlib_buffer (b);
ASSERT (rte_mbuf_refcnt_read (mb) == 1);
rte_pktmbuf_free (mb);
}
vec_free (f->name);
- vec_free (f->unaligned_buffers);
- vec_free (f->aligned_buffers);
+ vec_free (f->buffers);
}
/* Add buffer free list. */
if (PREDICT_FALSE (rmp == 0))
return 0;
- vlib_buffer_free_list_trim_aligned (fl);
-
/* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->aligned_buffers);
+ n = min_free_buffers - vec_len (fl->buffers);
if (n <= 0)
return min_free_buffers;
/* Always allocate round number of buffers. */
- n = round_pow2 (n, BUFFERS_PER_COPY);
+ n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
/* Always allocate new buffers in reasonably large sized chunks. */
n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
b = vlib_buffer_from_rte_mbuf (mb);
bi = vlib_get_buffer_index (vm, b);
- vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t));
+ vec_add1_aligned (fl->buffers, bi, CLIB_CACHE_LINE_BYTES);
n_alloc++;
n_remaining--;
return n;
}
-always_inline uword
-copy_alignment (u32 * x)
-{
- return (pointer_to_uword (x) / sizeof (x[0])) % BUFFERS_PER_COPY;
-}
-
static u32
alloc_from_free_list (vlib_main_t * vm,
vlib_buffer_free_list_t * free_list,
u32 * alloc_buffers, u32 n_alloc_buffers)
{
- u32 *dst, *u_src;
- uword u_len, n_left;
- uword n_unaligned_start, n_unaligned_end, n_filled;
+ u32 *dst, *src;
+ uword len, n_filled;
- n_left = n_alloc_buffers;
dst = alloc_buffers;
- n_unaligned_start = ((BUFFERS_PER_COPY - copy_alignment (dst))
- & (BUFFERS_PER_COPY - 1));
n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
if (n_filled == 0)
return 0;
- n_left = n_filled < n_left ? n_filled : n_left;
- n_alloc_buffers = n_left;
-
- if (n_unaligned_start >= n_left)
- {
- n_unaligned_start = n_left;
- n_unaligned_end = 0;
- }
- else
- n_unaligned_end = copy_alignment (dst + n_alloc_buffers);
-
- vlib_buffer_free_list_fill_unaligned (vm, free_list,
- n_unaligned_start + n_unaligned_end);
-
- u_len = vec_len (free_list->unaligned_buffers);
- u_src = free_list->unaligned_buffers + u_len - 1;
-
- if (n_unaligned_start)
- {
- uword n_copy = n_unaligned_start;
- if (n_copy > n_left)
- n_copy = n_left;
- n_left -= n_copy;
-
- while (n_copy > 0)
- {
- *dst++ = *u_src--;
- n_copy--;
- u_len--;
- }
-
- /* Now dst should be aligned. */
- if (n_left > 0)
- ASSERT (pointer_to_uword (dst) % sizeof (vlib_copy_unit_t) == 0);
- }
-
- /* Aligned copy. */
- {
- vlib_copy_unit_t *d, *s;
- uword n_copy;
-
- if (vec_len (free_list->aligned_buffers) <
- ((n_left / BUFFERS_PER_COPY) * BUFFERS_PER_COPY))
- abort ();
-
- n_copy = n_left / BUFFERS_PER_COPY;
- n_left = n_left % BUFFERS_PER_COPY;
-
- /* Remove buffers from aligned free list. */
- _vec_len (free_list->aligned_buffers) -= n_copy * BUFFERS_PER_COPY;
-
- s = (vlib_copy_unit_t *) vec_end (free_list->aligned_buffers);
- d = (vlib_copy_unit_t *) dst;
+ len = vec_len (free_list->buffers);
+ ASSERT (len >= n_alloc_buffers);
- /* Fast path loop. */
- while (n_copy >= 4)
- {
- d[0] = s[0];
- d[1] = s[1];
- d[2] = s[2];
- d[3] = s[3];
- n_copy -= 4;
- s += 4;
- d += 4;
- }
-
- while (n_copy >= 1)
- {
- d[0] = s[0];
- n_copy -= 1;
- s += 1;
- d += 1;
- }
-
- dst = (void *) d;
- }
-
- /* Unaligned copy. */
- ASSERT (n_unaligned_end == n_left);
- while (n_left > 0)
- {
- *dst++ = *u_src--;
- n_left--;
- u_len--;
- }
+ src = free_list->buffers + len - n_alloc_buffers;
+ clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
- if (!free_list->unaligned_buffers)
- ASSERT (u_len == 0);
- else
- _vec_len (free_list->unaligned_buffers) = u_len;
+ _vec_len (free_list->buffers) -= n_alloc_buffers;
return n_alloc_buffers;
}
* Note: this could be sped up if the node index were stuffed into
* the freelist itself.
*/
- if (vec_len (fl->aligned_buffers) > 0)
+ if (vec_len (fl->buffers) > 0)
{
- bi0 = fl->aligned_buffers[0];
- b0 = vlib_get_buffer (vm, bi0);
- ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count);
- feature_node_index = ctx->recycle_node_index;
- }
- else if (vec_len (fl->unaligned_buffers) > 0)
- {
- bi0 = fl->unaligned_buffers[0];
+ bi0 = fl->buffers[0];
b0 = vlib_get_buffer (vm, bi0);
ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count);
feature_node_index = ctx->recycle_node_index;
}
- /* aligned, unaligned buffers */
+ /* buffers */
for (i = 0; i < 2; i++)
{
if (i == 0)
{
- from = fl->aligned_buffers;
- n_left_from = vec_len (from);
- }
- else
- {
- from = fl->unaligned_buffers;
+ from = fl->buffers;
n_left_from = vec_len (from);
}
}
}
- vec_reset_length (fl->aligned_buffers);
- vec_reset_length (fl->unaligned_buffers);
+ vec_reset_length (fl->buffers);
if (f)
{
Code Review / vpp.git / commitdiff