always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
- uword l = b->current_length + b->total_length_not_including_first_buffer;
- if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
- | VLIB_BUFFER_TOTAL_LENGTH_VALID))
- == VLIB_BUFFER_NEXT_PRESENT))
- return vlib_buffer_length_in_chain_slow_path (vm, b);
- return l;
+ uword len = b->current_length;
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return len;
+
+ if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
+ return len + b->total_length_not_including_first_buffer;
+
+ return vlib_buffer_length_in_chain_slow_path (vm, b);
}
/** \brief Get length in bytes of the buffer index buffer chain
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (vlib_get_thread_index () == 0);
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
return p ? p[0] : VLIB_BUFFER_UNKNOWN;
}
vlib_buffer_known_state_t state)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (vlib_get_thread_index () == 0);
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
hash_set (bm->buffer_known_hash, buffer_index, state);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
}
/* Validates sanity of a single buffer.
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id);
-
/** \brief Allocate buffers into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
return round_pow2 (size, sizeof (vlib_buffer_t));
}
+always_inline u32
+vlib_buffer_get_free_list_index (vlib_buffer_t * b)
+{
+ return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+always_inline void
+vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index)
+{
+ /* if there is an need for more free lists we should consider
+ storig data in the 2nd cacheline */
+ ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1);
+ ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK);
+
+ b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+ b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
/** \brief Allocate buffers from specific freelist into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
vlib_buffer_main_t *bm = vm->buffer_main;
u32 i;
- *index = i = b->free_list_index;
+ *index = i = vlib_buffer_get_free_list_index (b);
return pool_elt_at_index (bm->buffer_free_list_pool, i);
}
}
n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
- s->free_list_index);
+ vlib_buffer_get_free_list_index
+ (s));
if (PREDICT_FALSE (n_buffers == 0))
{
buffers[0] = src_buffer;
vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
d->current_data = s->current_data;
d->current_length = head_end_offset;
- d->free_list_index = s->free_list_index;
+ vlib_buffer_set_free_list_index (d,
+ vlib_buffer_get_free_list_index (s));
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer + s->current_length -
head_end_offset;
vlib_buffer_t * tail)
{
ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
- ASSERT (head->free_list_index == tail->free_list_index);
+ ASSERT (vlib_buffer_get_free_list_index (head) ==
+ vlib_buffer_get_free_list_index (tail));
head->flags |= VLIB_BUFFER_NEXT_PRESENT;
head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
CLIB_CACHE_LINE_BYTES * 2);
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
STRUCT_MARK_PTR (src, template_start),
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
- ASSERT (dst->total_length_not_including_first_buffer == 0);
+ /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
+ /* total_length_not_including_first_buffer is not in the template anymore
+ * so it may actually not zeroed for some buffers. One option is to
+ * uncomment the line lower (comes at a cost), the other, is to just not
+ * care */
+ /* dst->total_length_not_including_first_buffer = 0; */
ASSERT (dst->n_add_refs == 0);
}
u32 buffer_index, u8 do_init)
{
vlib_buffer_t *b;
+ u32 i;
b = vlib_get_buffer (vm, buffer_index);
if (PREDICT_TRUE (do_init))
vlib_buffer_init_for_free_list (b, f);
vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+
+ if (vec_len (f->buffers) > 3 * VLIB_FRAME_SIZE)
+ {
+ /* keep last stored buffers, as they are more likely hot in the cache */
+ for (i = 0; i < VLIB_FRAME_SIZE; i++)
+ vm->os_physmem_free (vlib_get_buffer (vm, i));
+ vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+ }
}
always_inline void
vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
STRUCT_MARK_PTR (src, template_start),
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
ASSERT (dst0->total_length_not_including_first_buffer == 0);
Code Review / vpp.git / blobdiff