always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
- << CLIB_LOG2_CACHE_LINE_BYTES);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+ ASSERT (offset < bm->buffer_mem_size);
+
+ return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
/** \brief Translate buffer pointer into buffer index
@param p - (void *) buffer pointer
@return - (u32) buffer index
*/
+
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+ ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+ ASSERT (offset < bm->buffer_mem_size);
ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}
always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
- uword l = b->current_length + b->total_length_not_including_first_buffer;
- if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
- | VLIB_BUFFER_TOTAL_LENGTH_VALID))
- == VLIB_BUFFER_NEXT_PRESENT))
- return vlib_buffer_length_in_chain_slow_path (vm, b);
- return l;
+ uword len = b->current_length;
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return len;
+
+ if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
+ return len + b->total_length_not_including_first_buffer;
+
+ return vlib_buffer_length_in_chain_slow_path (vm, b);
}
/** \brief Get length in bytes of the buffer index buffer chain
always_inline u64
vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_offset_to_physical (&vm->physmem_main,
+ vlib_physmem_region_index_t pri;
+ vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
+ pri = vm->buffer_main->buffer_pools[b->buffer_pool_index].physmem_region;
+ return vlib_physmem_offset_to_physical (vm, pri,
(((uword) buffer_index) <<
CLIB_LOG2_CACHE_LINE_BYTES) +
STRUCT_OFFSET_OF (vlib_buffer_t,
VLIB_BUFFER_KNOWN_ALLOCATED,
} vlib_buffer_known_state_t;
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+ uword n_buffers,
+ vlib_buffer_known_state_t
+ expected_state);
+
always_inline vlib_buffer_known_state_t
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
return p ? p[0] : VLIB_BUFFER_UNKNOWN;
}
vlib_buffer_known_state_t state)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
hash_set (bm->buffer_known_hash, buffer_index, state);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
}
/* Validates sanity of a single buffer.
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id);
+always_inline u32
+vlib_buffer_round_size (u32 size)
+{
+ return round_pow2 (size, sizeof (vlib_buffer_t));
+}
-/** \brief Allocate buffers into supplied array
+always_inline u32
+vlib_buffer_get_free_list_index (vlib_buffer_t * b)
+{
+ return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+always_inline void
+vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index)
+{
+ /* if there is an need for more free lists we should consider
+ storig data in the 2nd cacheline */
+ ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1);
+ ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK);
+
+ b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+ b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+/** \brief Allocate buffers from specific freelist into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param buffers - (u32 * ) buffer index array
less than the number requested or zero
*/
always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
+ u32 * buffers,
+ u32 n_buffers, u32 free_list_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_free_list_t *fl;
+ u32 *src;
+ uword len;
- ASSERT (bm->cb.vlib_buffer_alloc_cb);
+ ASSERT (bm->cb.vlib_buffer_fill_free_list_cb);
- return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
-}
+ fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
-always_inline u32
-vlib_buffer_round_size (u32 size)
-{
- return round_pow2 (size, sizeof (vlib_buffer_t));
+ len = vec_len (fl->buffers);
+
+ if (PREDICT_FALSE (len < n_buffers))
+ {
+ bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
+ len = vec_len (fl->buffers);
+
+ /* even if fill free list didn't manage to refill free list
+ we should give what we have */
+ n_buffers = clib_min (len, n_buffers);
+
+ /* following code is intentionaly duplicated to allow compiler
+ to optimize fast path when n_buffers is constant value */
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
+
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+
+ return n_buffers;
+ }
+
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
+
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+
+ return n_buffers;
}
-/** \brief Allocate buffers from specific freelist into supplied array
+/** \brief Allocate buffers into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param buffers - (u32 * ) buffer index array
less than the number requested or zero
*/
always_inline u32
-vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
-
- return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
- free_list_index);
+ return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
}
/** \brief Free buffers
u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
char *fmt, ...);
+/* Merge two free lists */
+void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
+ vlib_buffer_free_list_t * src);
+
+/* Make sure we have at least given number of unaligned buffers. */
+void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
+ vlib_buffer_free_list_t *
+ free_list,
+ uword n_unaligned_buffers);
+
+always_inline u32
+vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ size = vlib_buffer_round_size (size);
+ uword *p = hash_get (bm->free_list_by_size, size);
+ return p ? p[0] : ~0;
+}
+
+always_inline vlib_buffer_free_list_t *
+vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b,
+ u32 * index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ u32 i;
+
+ *index = i = vlib_buffer_get_free_list_index (b);
+ return pool_elt_at_index (bm->buffer_free_list_pool, i);
+}
+
always_inline vlib_buffer_free_list_t *
vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
{
}
}
-always_inline void *
-vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
- uword n_bytes, uword alignment)
-{
- void *r =
- vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
- if (!r)
- *error =
- clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
- n_bytes);
- else
- *error = 0;
- return r;
-}
-
-/* By default allocate I/O memory with cache line alignment. */
-always_inline void *
-vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
-{
- return vlib_physmem_alloc_aligned (vm, error, n_bytes,
- CLIB_CACHE_LINE_BYTES);
-}
-
-always_inline void
-vlib_physmem_free (vlib_main_t * vm, void *mem)
-{
- return vm->os_physmem_free (mem);
-}
-
-always_inline u64
-vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem)
-{
- vlib_physmem_main_t *pm = &vm->physmem_main;
- uword o = pointer_to_uword (mem) - pm->virtual.start;
- return vlib_physmem_offset_to_physical (pm, o);
-}
-
/* Append given data to end of buffer, possibly allocating new buffers. */
u32 vlib_buffer_add_data (vlib_main_t * vm,
u32 free_list_index,
{
vlib_buffer_t *s, *d, *fd;
uword n_alloc, n_buffers = 1;
- u32 *new_buffers = 0;
u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
int i;
n_buffers++;
s = vlib_get_buffer (vm, s->next_buffer);
}
+ u32 new_buffers[n_buffers];
- vec_validate (new_buffers, n_buffers - 1);
n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
- ASSERT (n_alloc == n_buffers);
+
+ /* No guarantee that we'll get all the buffers we asked for */
+ if (PREDICT_FALSE (n_alloc < n_buffers))
+ {
+ if (n_alloc > 0)
+ vlib_buffer_free (vm, new_buffers, n_alloc);
+ return 0;
+ }
/* 1st segment */
s = b;
return fd;
}
+/** \brief Create multiple clones of buffer and store them in the supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param src_buffer - (u32) source buffer index
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u8) number of buffer clones requested
+ @param head_end_offset - (u16) offset relative to current position
+ where packet head ends
+ @return - (u8) number of buffers actually cloned, may be
+ less than the number requested or zero
+*/
+
+always_inline u8
+vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+ u8 n_buffers, u16 head_end_offset)
+{
+ u8 i;
+ vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
+
+ ASSERT (s->n_add_refs == 0);
+ ASSERT (n_buffers);
+
+ if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
+ {
+ buffers[0] = src_buffer;
+ for (i = 1; i < n_buffers; i++)
+ {
+ vlib_buffer_t *d;
+ d = vlib_buffer_copy (vm, s);
+ if (d == 0)
+ return i;
+ buffers[i] = vlib_get_buffer_index (vm, d);
+
+ }
+ return n_buffers;
+ }
+
+ n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+ vlib_buffer_get_free_list_index
+ (s));
+ if (PREDICT_FALSE (n_buffers == 0))
+ {
+ buffers[0] = src_buffer;
+ return 1;
+ }
+
+ for (i = 0; i < n_buffers; i++)
+ {
+ vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
+ d->current_data = s->current_data;
+ d->current_length = head_end_offset;
+ vlib_buffer_set_free_list_index (d,
+ vlib_buffer_get_free_list_index (s));
+ d->total_length_not_including_first_buffer =
+ s->total_length_not_including_first_buffer + s->current_length -
+ head_end_offset;
+ d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
+ d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
+ clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
+ head_end_offset);
+ d->next_buffer = src_buffer;
+ }
+ vlib_buffer_advance (s, head_end_offset);
+ s->n_add_refs = n_buffers - 1;
+ while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ s = vlib_get_buffer (vm, s->next_buffer);
+ s->n_add_refs = n_buffers - 1;
+ }
+
+ return n_buffers;
+}
+
+/** \brief Attach cloned tail to the buffer
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param head - (vlib_buffer_t *) head buffer
+ @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
+*/
+
+always_inline void
+vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
+ vlib_buffer_t * tail)
+{
+ ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
+ ASSERT (vlib_buffer_get_free_list_index (head) ==
+ vlib_buffer_get_free_list_index (tail));
+
+ head->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
+ head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
+ head->next_buffer = vlib_get_buffer_index (vm, tail);
+ head->total_length_not_including_first_buffer = tail->current_length +
+ tail->total_length_not_including_first_buffer;
+
+next_segment:
+ __sync_add_and_fetch (&tail->n_add_refs, 1);
+
+ if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ tail = vlib_get_buffer (vm, tail->next_buffer);
+ goto next_segment;
+ }
+}
+
/* Initializes the buffer as an empty packet with no chained buffers. */
always_inline void
vlib_buffer_chain_init (vlib_buffer_t * first)
return n;
}
-typedef union
-{
- vlib_buffer_t b;
- vlib_copy_unit_t i[sizeof (vlib_buffer_t) / sizeof (vlib_copy_unit_t)];
-}
-vlib_buffer_union_t;
-
/* Set a buffer quickly into "uninitialized" state. We want this to
be extremely cheap and arrange for all fields that need to be
initialized to be in the first 128 bits of the buffer. */
always_inline void
-vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
+vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst = (vlib_buffer_union_t *) _dst;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure vlib_buffer_t is cacheline aligned and sized */
ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
CLIB_CACHE_LINE_BYTES * 2);
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
- /* Copy template from src->current_data thru src->free_list_index */
- dst->i[0] = src->i[0];
- if (1 * sizeof (dst->i[0]) < 16)
- dst->i[1] = src->i[1];
- if (2 * sizeof (dst->i[0]) < 16)
- dst->i[2] = src->i[2];
+ clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) ASSERT (dst->b.f == src->b.f)
+#define _(f) ASSERT (dst->f == src->f);
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
- ASSERT (dst->b.total_length_not_including_first_buffer == 0);
+ /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
+ /* total_length_not_including_first_buffer is not in the template anymore
+ * so it may actually not zeroed for some buffers. One option is to
+ * uncomment the line lower (comes at a cost), the other, is to just not
+ * care */
+ /* dst->total_length_not_including_first_buffer = 0; */
+ ASSERT (dst->n_add_refs == 0);
}
always_inline void
-vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
- vlib_buffer_t * _dst1,
+vlib_buffer_add_to_free_list (vlib_main_t * vm,
+ vlib_buffer_free_list_t * f,
+ u32 buffer_index, u8 do_init)
+{
+ vlib_buffer_t *b;
+ b = vlib_get_buffer (vm, buffer_index);
+ if (PREDICT_TRUE (do_init))
+ vlib_buffer_init_for_free_list (b, f);
+ vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+
+ if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
+ {
+ vlib_buffer_free_list_t *mf;
+ mf = vlib_buffer_get_free_list (vlib_mains[0], f->index);
+ clib_spinlock_lock (&mf->global_buffers_lock);
+ /* keep last stored buffers, as they are more likely hot in the cache */
+ vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE,
+ CLIB_CACHE_LINE_BYTES);
+ vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+ f->n_alloc -= VLIB_FRAME_SIZE;
+ clib_spinlock_unlock (&mf->global_buffers_lock);
+ }
+}
+
+always_inline void
+vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0,
+ vlib_buffer_t * dst1,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst0 = (vlib_buffer_union_t *) _dst0;
- vlib_buffer_union_t *dst1 = (vlib_buffer_union_t *) _dst1;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
- /* Copy template from src->current_data thru src->free_list_index */
- dst0->i[0] = dst1->i[0] = src->i[0];
- if (1 * sizeof (dst0->i[0]) < 16)
- dst0->i[1] = dst1->i[1] = src->i[1];
- if (2 * sizeof (dst0->i[0]) < 16)
- dst0->i[2] = dst1->i[2] = src->i[2];
+ clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst0->n_add_refs = src->n_add_refs;
+ dst1->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) ASSERT (dst0->b.f == src->b.f && dst1->b.f == src->b.f)
+#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
- ASSERT (dst0->b.total_length_not_including_first_buffer == 0);
- ASSERT (dst1->b.total_length_not_including_first_buffer == 0);
+
+ ASSERT (dst0->total_length_not_including_first_buffer == 0);
+ ASSERT (dst1->total_length_not_including_first_buffer == 0);
+ ASSERT (dst0->n_add_refs == 0);
+ ASSERT (dst1->n_add_refs == 0);
}
#if CLIB_DEBUG > 0
Code Review / vpp.git / blobdiff