always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
- << CLIB_LOG2_CACHE_LINE_BYTES);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+ ASSERT (offset < bm->buffer_mem_size);
+
+ return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
/** \brief Translate buffer pointer into buffer index
@param p - (void *) buffer pointer
@return - (u32) buffer index
*/
+
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+ ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+ ASSERT (offset < bm->buffer_mem_size);
ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}
always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
- uword l = b->current_length + b->total_length_not_including_first_buffer;
- if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
- | VLIB_BUFFER_TOTAL_LENGTH_VALID))
- == VLIB_BUFFER_NEXT_PRESENT))
- return vlib_buffer_length_in_chain_slow_path (vm, b);
- return l;
+ uword len = b->current_length;
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return len;
+
+ if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
+ return len + b->total_length_not_including_first_buffer;
+
+ return vlib_buffer_length_in_chain_slow_path (vm, b);
}
/** \brief Get length in bytes of the buffer index buffer chain
always_inline u64
vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_offset_to_physical (&vm->physmem_main,
- (((uword) buffer_index) <<
- CLIB_LOG2_CACHE_LINE_BYTES) +
- STRUCT_OFFSET_OF (vlib_buffer_t,
- data));
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
+ vlib_buffer_pool_t *pool = vec_elt_at_index (bm->buffer_pools,
+ b->buffer_pool_index);
+
+ return vlib_physmem_virtual_to_physical (vm, pool->physmem_region, b->data);
}
/** \brief Prefetch buffer metadata by buffer index
VLIB_BUFFER_KNOWN_ALLOCATED,
} vlib_buffer_known_state_t;
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+ uword n_buffers,
+ vlib_buffer_known_state_t
+ expected_state);
+
always_inline vlib_buffer_known_state_t
-vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
+vlib_buffer_is_known (u32 buffer_index)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ vlib_buffer_main_t *bm = vlib_global_main.buffer_main;
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
return p ? p[0] : VLIB_BUFFER_UNKNOWN;
}
always_inline void
-vlib_buffer_set_known_state (vlib_main_t * vm,
- u32 buffer_index,
+vlib_buffer_set_known_state (u32 buffer_index,
vlib_buffer_known_state_t state)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ vlib_buffer_main_t *bm = vlib_global_main.buffer_main;
+
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
hash_set (bm->buffer_known_hash, buffer_index, state);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
}
/* Validates sanity of a single buffer.
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id);
+always_inline u32
+vlib_buffer_round_size (u32 size)
+{
+ return round_pow2 (size, sizeof (vlib_buffer_t));
+}
-/** \brief Allocate buffers into supplied array
+always_inline u32
+vlib_buffer_get_free_list_index (vlib_buffer_t * b)
+{
+ return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+always_inline void
+vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index)
+{
+ /* if there is an need for more free lists we should consider
+ storig data in the 2nd cacheline */
+ ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1);
+ ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK);
+
+ b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+ b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+/** \brief Allocate buffers from specific freelist into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param buffers - (u32 * ) buffer index array
less than the number requested or zero
*/
always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
+ u32 * buffers,
+ u32 n_buffers, u32 free_list_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_free_list_t *fl;
+ u32 *src;
+ uword len;
+
+ ASSERT (bm->cb.vlib_buffer_fill_free_list_cb);
+
+ fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
+
+ len = vec_len (fl->buffers);
+
+ if (PREDICT_FALSE (len < n_buffers))
+ {
+ bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
+ len = vec_len (fl->buffers);
+
+ /* even if fill free list didn't manage to refill free list
+ we should give what we have */
+ n_buffers = clib_min (len, n_buffers);
+
+ /* following code is intentionaly duplicated to allow compiler
+ to optimize fast path when n_buffers is constant value */
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
+
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+
+ return n_buffers;
+ }
+
+ src = fl->buffers + len - n_buffers;
+ clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ _vec_len (fl->buffers) -= n_buffers;
- ASSERT (bm->cb.vlib_buffer_alloc_cb);
+ /* Verify that buffers are known free. */
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
- return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
+ return n_buffers;
}
+/** \brief Allocate buffers into supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u32) number of buffers requested
+ @return - (u32) number of buffers actually allocated, may be
+ less than the number requested or zero
+*/
always_inline u32
-vlib_buffer_round_size (u32 size)
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
{
- return round_pow2 (size, sizeof (vlib_buffer_t));
+ return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
}
-/** \brief Allocate buffers from specific freelist into supplied array
+/** \brief Allocate buffers into ring
@param vm - (vlib_main_t *) vlib main data structure pointer
- @param buffers - (u32 * ) buffer index array
+ @param buffers - (u32 * ) buffer index ring
+ @param start - (u32) first slot in the ring
+ @param ring_size - (u32) ring size
@param n_buffers - (u32) number of buffers requested
@return - (u32) number of buffers actually allocated, may be
less than the number requested or zero
*/
always_inline u32
-vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
+vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
+ u32 n_alloc;
+
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ return vlib_buffer_alloc (vm, ring + start, n_buffers);
- ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+ n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
- return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
- free_list_index);
+ if (PREDICT_TRUE (n_alloc == ring_size - start))
+ n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
+
+ return n_alloc;
}
/** \brief Free buffers
vlib_buffer_main_t *bm = vm->buffer_main;
u32 i;
- *index = i = b->free_list_index;
+ *index = i = vlib_buffer_get_free_list_index (b);
return pool_elt_at_index (bm->buffer_free_list_pool, i);
}
}
}
-always_inline void *
-vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
- uword n_bytes, uword alignment)
-{
- void *r =
- vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
- if (!r)
- *error =
- clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
- n_bytes);
- else
- *error = 0;
- return r;
-}
-
-/* By default allocate I/O memory with cache line alignment. */
-always_inline void *
-vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
-{
- return vlib_physmem_alloc_aligned (vm, error, n_bytes,
- CLIB_CACHE_LINE_BYTES);
-}
-
-always_inline void
-vlib_physmem_free (vlib_main_t * vm, void *mem)
-{
- return vm->os_physmem_free (mem);
-}
-
-always_inline u64
-vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem)
-{
- vlib_physmem_main_t *pm = &vm->physmem_main;
- uword o = pointer_to_uword (mem) - pm->virtual.start;
- return vlib_physmem_offset_to_physical (pm, o);
-}
-
/* Append given data to end of buffer, possibly allocating new buffers. */
u32 vlib_buffer_add_data (vlib_main_t * vm,
u32 free_list_index,
{
vlib_buffer_t *s, *d, *fd;
uword n_alloc, n_buffers = 1;
- u32 *new_buffers = 0;
u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
int i;
n_buffers++;
s = vlib_get_buffer (vm, s->next_buffer);
}
+ u32 new_buffers[n_buffers];
- vec_validate (new_buffers, n_buffers - 1);
n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
/* No guarantee that we'll get all the buffers we asked for */
{
if (n_alloc > 0)
vlib_buffer_free (vm, new_buffers, n_alloc);
- vec_free (new_buffers);
return 0;
}
d->flags = s->flags & flag_mask;
}
- vec_free (new_buffers);
return fd;
}
}
n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
- s->free_list_index);
+ vlib_buffer_get_free_list_index
+ (s));
if (PREDICT_FALSE (n_buffers == 0))
{
buffers[0] = src_buffer;
vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
d->current_data = s->current_data;
d->current_length = head_end_offset;
- d->free_list_index = s->free_list_index;
+ vlib_buffer_set_free_list_index (d,
+ vlib_buffer_get_free_list_index (s));
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer + s->current_length -
head_end_offset;
vlib_buffer_t * tail)
{
ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
- ASSERT (head->free_list_index == tail->free_list_index);
+ ASSERT (vlib_buffer_get_free_list_index (head) ==
+ vlib_buffer_get_free_list_index (tail));
head->flags |= VLIB_BUFFER_NEXT_PRESENT;
head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
CLIB_CACHE_LINE_BYTES * 2);
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) dst->f = src->f
+#define _(f) ASSERT (dst->f == src->f);
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
- dst->total_length_not_including_first_buffer = 0;
+ /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
+ /* total_length_not_including_first_buffer is not in the template anymore
+ * so it may actually not zeroed for some buffers. One option is to
+ * uncomment the line lower (comes at a cost), the other, is to just not
+ * care */
+ /* dst->total_length_not_including_first_buffer = 0; */
ASSERT (dst->n_add_refs == 0);
}
if (PREDICT_TRUE (do_init))
vlib_buffer_init_for_free_list (b, f);
vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+
+ if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
+ {
+ vlib_buffer_free_list_t *mf;
+ mf = vlib_buffer_get_free_list (vlib_mains[0], f->index);
+ clib_spinlock_lock (&mf->global_buffers_lock);
+ /* keep last stored buffers, as they are more likely hot in the cache */
+ vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE,
+ CLIB_CACHE_LINE_BYTES);
+ vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+ f->n_alloc -= VLIB_FRAME_SIZE;
+ clib_spinlock_unlock (&mf->global_buffers_lock);
+ }
}
always_inline void
vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst0->n_add_refs = src->n_add_refs;
+ dst1->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) dst0->f = src->f; dst1->f = src->f
+#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
- dst0->total_length_not_including_first_buffer = 0;
- dst1->total_length_not_including_first_buffer = 0;
+
+ ASSERT (dst0->total_length_not_including_first_buffer == 0);
+ ASSERT (dst1->total_length_not_including_first_buffer == 0);
ASSERT (dst0->n_add_refs == 0);
ASSERT (dst1->n_add_refs == 0);
}