always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
- << CLIB_LOG2_CACHE_LINE_BYTES);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+ ASSERT (offset < bm->buffer_mem_size);
+
+ return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
/** \brief Translate buffer pointer into buffer index
@param p - (void *) buffer pointer
@return - (u32) buffer index
*/
+
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+ ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+ ASSERT (offset < bm->buffer_mem_size);
ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}
always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
- uword l = b->current_length + b->total_length_not_including_first_buffer;
- if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
- | VLIB_BUFFER_TOTAL_LENGTH_VALID))
- == VLIB_BUFFER_NEXT_PRESENT))
- return vlib_buffer_length_in_chain_slow_path (vm, b);
- return l;
+ uword len = b->current_length;
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return len;
+
+ if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
+ return len + b->total_length_not_including_first_buffer;
+
+ return vlib_buffer_length_in_chain_slow_path (vm, b);
}
/** \brief Get length in bytes of the buffer index buffer chain
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ ASSERT (vlib_get_thread_index () == 0);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
return p ? p[0] : VLIB_BUFFER_UNKNOWN;
vlib_buffer_known_state_t state)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ ASSERT (vlib_get_thread_index () == 0);
hash_set (bm->buffer_known_hash, buffer_index, state);
}
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id);
-
/** \brief Allocate buffers into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
return round_pow2 (size, sizeof (vlib_buffer_t));
}
+always_inline u32
+vlib_buffer_get_free_list_index (vlib_buffer_t * b)
+{
+ return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
+always_inline void
+vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index)
+{
+ /* if there is an need for more free lists we should consider
+ storig data in the 2nd cacheline */
+ ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1);
+ ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK);
+
+ b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+ b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK;
+}
+
/** \brief Allocate buffers from specific freelist into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
vlib_buffer_main_t *bm = vm->buffer_main;
u32 i;
- *index = i = b->free_list_index;
+ *index = i = vlib_buffer_get_free_list_index (b);
return pool_elt_at_index (bm->buffer_free_list_pool, i);
}
{
vlib_buffer_t *s, *d, *fd;
uword n_alloc, n_buffers = 1;
- u32 *new_buffers = 0;
u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
int i;
n_buffers++;
s = vlib_get_buffer (vm, s->next_buffer);
}
+ u32 new_buffers[n_buffers];
- vec_validate (new_buffers, n_buffers - 1);
n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
- ASSERT (n_alloc == n_buffers);
+
+ /* No guarantee that we'll get all the buffers we asked for */
+ if (PREDICT_FALSE (n_alloc < n_buffers))
+ {
+ if (n_alloc > 0)
+ vlib_buffer_free (vm, new_buffers, n_alloc);
+ return 0;
+ }
/* 1st segment */
s = b;
return fd;
}
+/** \brief Create multiple clones of buffer and store them in the supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param src_buffer - (u32) source buffer index
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u8) number of buffer clones requested
+ @param head_end_offset - (u16) offset relative to current position
+ where packet head ends
+ @return - (u8) number of buffers actually cloned, may be
+ less than the number requested or zero
+*/
+
+always_inline u8
+vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+ u8 n_buffers, u16 head_end_offset)
+{
+ u8 i;
+ vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
+
+ ASSERT (s->n_add_refs == 0);
+ ASSERT (n_buffers);
+
+ if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
+ {
+ buffers[0] = src_buffer;
+ for (i = 1; i < n_buffers; i++)
+ {
+ vlib_buffer_t *d;
+ d = vlib_buffer_copy (vm, s);
+ if (d == 0)
+ return i;
+ buffers[i] = vlib_get_buffer_index (vm, d);
+
+ }
+ return n_buffers;
+ }
+
+ n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+ vlib_buffer_get_free_list_index
+ (s));
+ if (PREDICT_FALSE (n_buffers == 0))
+ {
+ buffers[0] = src_buffer;
+ return 1;
+ }
+
+ for (i = 0; i < n_buffers; i++)
+ {
+ vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
+ d->current_data = s->current_data;
+ d->current_length = head_end_offset;
+ vlib_buffer_set_free_list_index (d,
+ vlib_buffer_get_free_list_index (s));
+ d->total_length_not_including_first_buffer =
+ s->total_length_not_including_first_buffer + s->current_length -
+ head_end_offset;
+ d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
+ d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
+ clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
+ head_end_offset);
+ d->next_buffer = src_buffer;
+ }
+ vlib_buffer_advance (s, head_end_offset);
+ s->n_add_refs = n_buffers - 1;
+ while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ s = vlib_get_buffer (vm, s->next_buffer);
+ s->n_add_refs = n_buffers - 1;
+ }
+
+ return n_buffers;
+}
+
+/** \brief Attach cloned tail to the buffer
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param head - (vlib_buffer_t *) head buffer
+ @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
+*/
+
+always_inline void
+vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
+ vlib_buffer_t * tail)
+{
+ ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
+ ASSERT (vlib_buffer_get_free_list_index (head) ==
+ vlib_buffer_get_free_list_index (tail));
+
+ head->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
+ head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
+ head->next_buffer = vlib_get_buffer_index (vm, tail);
+ head->total_length_not_including_first_buffer = tail->current_length +
+ tail->total_length_not_including_first_buffer;
+
+next_segment:
+ __sync_add_and_fetch (&tail->n_add_refs, 1);
+
+ if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ tail = vlib_get_buffer (vm, tail->next_buffer);
+ goto next_segment;
+ }
+}
+
/* Initializes the buffer as an empty packet with no chained buffers. */
always_inline void
vlib_buffer_chain_init (vlib_buffer_t * first)
CLIB_CACHE_LINE_BYTES * 2);
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) dst->f = src->f
+#define _(f) ASSERT (dst->f == src->f);
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
ASSERT (dst->total_length_not_including_first_buffer == 0);
+ ASSERT (dst->n_add_refs == 0);
}
always_inline void
vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+ ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+
+ /* Not in the first 16 octets. */
+ dst0->n_add_refs = src->n_add_refs;
+ dst1->n_add_refs = src->n_add_refs;
/* Make sure it really worked. */
-#define _(f) dst0->f = src->f; dst1->f = src->f
+#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
_(current_data);
_(current_length);
_(flags);
- _(free_list_index);
#undef _
+
ASSERT (dst0->total_length_not_including_first_buffer == 0);
ASSERT (dst1->total_length_not_including_first_buffer == 0);
+ ASSERT (dst0->n_add_refs == 0);
+ ASSERT (dst1->n_add_refs == 0);
}
#if CLIB_DEBUG > 0
Code Review / vpp.git / blobdiff