always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
ASSERT (offset < bm->buffer_mem_size);
return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
+static_always_inline void
+vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
+{
+ clib_memcpy_fast (dst, src, n_indices * sizeof (u32));
+}
+
+static_always_inline void
+vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
+{
+ clib_memcpy_fast (b, bt, STRUCT_OFFSET_OF (vlib_buffer_t, template_end));
+}
+
/** \brief Translate array of buffer indices into buffer pointers with offset
@param vm - (vlib_main_t *) vlib main data structure pointer
vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
i32 offset)
{
+ uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
#ifdef CLIB_HAVE_VEC256
- u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset);
+ u64x4 off = u64x4_splat (buffer_mem_start + offset);
/* if count is not const, compiler will not unroll while loop
se we maintain two-in-parallel variant */
while (count >= 8)
/* shift and add to get vlib_buffer_t pointer */
u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
#elif defined (CLIB_HAVE_VEC128)
- u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset);
+ u64x2 off = u64x2_splat (buffer_mem_start + offset);
u32x4 bi4 = u32x4_load_unaligned (bi);
u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
#if defined (__aarch64__)
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
ASSERT (offset < bm->buffer_mem_size);
{
#ifdef CLIB_HAVE_VEC256
u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
- u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset);
+ u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
while (count >= 8)
{
{
b = vlib_get_buffer (vm, buffer_index);
l = b->current_length;
- clib_memcpy (contents + content_len, b->data + b->current_data, l);
+ clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
content_len += l;
if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
break;
vlib_prefetch_buffer_header (_b, type); \
} while (0)
-#if 0
-/* Iterate over known allocated vlib bufs. You probably do not want
- * to do this!
- @param vm the vlib_main_t
- @param bi found allocated buffer index
- @param body operation to perform on buffer index
- function executes body for each allocated buffer index
- */
-#define vlib_buffer_foreach_allocated(vm,bi,body) \
-do { \
- vlib_main_t * _vmain = (vm); \
- vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
- hash_pair_t * _vbpair; \
- hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
- if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
- (bi) = _vbpair->key; \
- body; \
- } \
- })); \
-} while (0)
-#endif
-
typedef enum
{
/* Index is unknown. */
expected_state);
always_inline vlib_buffer_known_state_t
-vlib_buffer_is_known (u32 buffer_index)
+vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
clib_spinlock_lock (&bm->buffer_known_hash_lockp);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
}
always_inline void
-vlib_buffer_set_known_state (u32 buffer_index,
+vlib_buffer_set_known_state (vlib_main_t * vm, u32 buffer_index,
vlib_buffer_known_state_t state)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
clib_spinlock_lock (&bm->buffer_known_hash_lockp);
hash_set (bm->buffer_known_hash, buffer_index, state);
return round_pow2 (size, sizeof (vlib_buffer_t));
}
-always_inline vlib_buffer_free_list_index_t
-vlib_buffer_get_free_list_index (vlib_buffer_t * b)
-{
- if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NON_DEFAULT_FREELIST))
- return b->free_list_index;
-
- return 0;
-}
-
-always_inline void
-vlib_buffer_set_free_list_index (vlib_buffer_t * b,
- vlib_buffer_free_list_index_t index)
-{
- if (PREDICT_FALSE (index))
- {
- b->flags |= VLIB_BUFFER_NON_DEFAULT_FREELIST;
- b->free_list_index = index;
- }
- else
- b->flags &= ~VLIB_BUFFER_NON_DEFAULT_FREELIST;
-}
-
/** \brief Allocate buffers from specific freelist into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
u32 n_buffers,
vlib_buffer_free_list_index_t index)
{
- vlib_buffer_main_t *bm = &buffer_main;
vlib_buffer_free_list_t *fl;
+ vlib_buffer_main_t *bm = vm->buffer_main;
u32 *src;
uword len;
/* following code is intentionaly duplicated to allow compiler
to optimize fast path when n_buffers is constant value */
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ vlib_buffer_copy_indices (buffers, src, n_buffers);
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
}
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ vlib_buffer_copy_indices (buffers, src, n_buffers);
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
/* number of buffers to free */
u32 n_buffers)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
ASSERT (bm->cb.vlib_buffer_free_cb);
/* number of buffers to free */
u32 n_buffers)
{
- vlib_buffer_main_t *bm = &buffer_main;
+ vlib_buffer_main_t *bm = vm->buffer_main;
ASSERT (bm->cb.vlib_buffer_free_no_next_cb);
vlib_buffer_free_list_index_t vlib_buffer_create_free_list (vlib_main_t * vm,
u32 n_data_bytes,
char *fmt, ...);
-always_inline void
-vlib_buffer_delete_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_index_t free_list_index)
-{
- vlib_buffer_main_t *bm = &buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_delete_free_list_cb);
-
- bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
-}
-
-/* Make sure we have at least given number of unaligned buffers. */
-void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
- vlib_buffer_free_list_t *
- free_list,
- uword n_unaligned_buffers);
-
-always_inline vlib_buffer_free_list_t *
-vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b,
- vlib_buffer_free_list_index_t * index)
-{
- vlib_buffer_free_list_index_t i;
-
- *index = i = vlib_buffer_get_free_list_index (b);
- return pool_elt_at_index (vm->buffer_free_list_pool, i);
-}
always_inline vlib_buffer_free_list_t *
vlib_buffer_get_free_list (vlib_main_t * vm,
return f->n_data_bytes;
}
-void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
-
-/* Reasonably fast buffer copy routine. */
-always_inline void
-vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
-{
- while (n >= 4)
- {
- dst[0] = src[0];
- dst[1] = src[1];
- dst[2] = src[2];
- dst[3] = src[3];
- dst += 4;
- src += 4;
- n -= 4;
- }
- while (n > 0)
- {
- dst[0] = src[0];
- dst += 1;
- src += 1;
- n -= 1;
- }
-}
-
/* Append given data to end of buffer, possibly allocating new buffers. */
-u32 vlib_buffer_add_data (vlib_main_t * vm,
- vlib_buffer_free_list_index_t free_list_index,
- u32 buffer_index, void *data, u32 n_data_bytes);
+int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
+ u32 n_data_bytes);
/* duplicate all buffers in chain */
always_inline vlib_buffer_t *
d->flags = s->flags & flag_mask;
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
/* next segments */
for (i = 1; i < n_buffers; i++)
d = vlib_get_buffer (vm, new_buffers[i]);
d->current_data = s->current_data;
d->current_length = s->current_length;
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
d->flags = s->flags & flag_mask;
}
return 1;
}
- n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
- vlib_buffer_get_free_list_index
- (s));
+ n_buffers = vlib_buffer_alloc (vm, buffers, n_buffers);
for (i = 0; i < n_buffers; i++)
{
vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
d->current_data = s->current_data;
d->current_length = head_end_offset;
- vlib_buffer_set_free_list_index (d,
- vlib_buffer_get_free_list_index (s));
-
d->total_length_not_including_first_buffer = s->current_length -
head_end_offset;
if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
}
d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
- clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
- head_end_offset);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), head_end_offset);
d->next_buffer = src_buffer;
}
vlib_buffer_advance (s, head_end_offset);
vlib_buffer_t * tail)
{
ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
- ASSERT (vlib_buffer_get_free_list_index (head) ==
- vlib_buffer_get_free_list_index (tail));
head->flags |= VLIB_BUFFER_NEXT_PRESENT;
head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
u16 len = clib_min (data_len,
n_buffer_bytes - last->current_length -
last->current_data);
- clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
- len);
+ clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
+ data, len);
vlib_buffer_chain_increase_length (first, last, len);
return len;
}
vlib_buffer_free_list_index_t free_list_index;
u32 *free_buffers;
+
+ u8 *name;
} vlib_packet_template_t;
void vlib_packet_template_get_packet_helper (vlib_main_t * vm,
vec_free (t->packet_data);
}
-always_inline u32
-unserialize_vlib_buffer_n_bytes (serialize_main_t * m)
+static_always_inline vlib_buffer_pool_t *
+vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index)
{
- serialize_stream_t *s = &m->stream;
- vlib_serialize_buffer_main_t *sm
- = uword_to_pointer (m->stream.data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vlib_main_t *vm = sm->vlib_main;
- u32 n, *f;
-
- n = s->n_buffer_bytes - s->current_buffer_index;
- if (sm->last_buffer != ~0)
- {
- vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer);
- while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
- {
- b = vlib_get_buffer (vm, b->next_buffer);
- n += b->current_length;
- }
- }
-
- /* *INDENT-OFF* */
- clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
- n += vlib_buffer_index_length_in_chain (vm, f[0]);
- }));
-/* *INDENT-ON* */
-
- return n;
-}
-
-/* Set a buffer quickly into "uninitialized" state. We want this to
- be extremely cheap and arrange for all fields that need to be
- initialized to be in the first 128 bits of the buffer. */
-always_inline void
-vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
- vlib_buffer_free_list_t * fl)
-{
- vlib_buffer_t *src = &fl->buffer_init_template;
-
- /* Make sure vlib_buffer_t is cacheline aligned and sized */
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
- CLIB_CACHE_LINE_BYTES);
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
- CLIB_CACHE_LINE_BYTES * 2);
-
- /* Make sure buffer template is sane. */
- ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
-
- clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
-
- /* Not in the first 16 octets. */
- dst->n_add_refs = src->n_add_refs;
- vlib_buffer_set_free_list_index (dst, fl->index);
-
- /* Make sure it really worked. */
-#define _(f) ASSERT (dst->f == src->f);
- _(current_data);
- _(current_length);
- _(flags);
-#undef _
- /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
- /* total_length_not_including_first_buffer is not in the template anymore
- * so it may actually not zeroed for some buffers. One option is to
- * uncomment the line lower (comes at a cost), the other, is to just not
- * care */
- /* dst->total_length_not_including_first_buffer = 0; */
- ASSERT (dst->n_add_refs == 0);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
}
always_inline void
vlib_buffer_free_list_t * f,
u32 buffer_index, u8 do_init)
{
- vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index);
+ vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index);
vlib_buffer_t *b;
b = vlib_get_buffer (vm, buffer_index);
if (PREDICT_TRUE (do_init))
- vlib_buffer_init_for_free_list (b, f);
+ clib_memset (b, 0, STRUCT_OFFSET_OF (vlib_buffer_t, template_end));
vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
return;
}
/* probe free list to find allocated buffer size to avoid overfill */
- vlib_buffer_free_list_index_t index;
- vlib_buffer_free_list_t *free_list =
- vlib_buffer_get_buffer_free_list (vm, first, &index);
+ vlib_buffer_free_list_t *free_list;
+
+ free_list = pool_elt_at_index (vm->buffer_free_list_pool,
+ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
u32 want_first_size = clib_min (VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE,
free_list->n_data_bytes -
vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer);
u32 need = want_first_size - first->current_length;
u32 amount_to_copy = clib_min (need, second->current_length);
- clib_memcpy (((u8 *) vlib_buffer_get_current (first)) +
- first->current_length,
- vlib_buffer_get_current (second), amount_to_copy);
+ clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) +
+ first->current_length,
+ vlib_buffer_get_current (second), amount_to_copy);
first->current_length += amount_to_copy;
- vlib_buffer_advance (second, amount_to_copy);
+ second->current_data += amount_to_copy;
+ second->current_length -= amount_to_copy;
if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
{
first->total_length_not_including_first_buffer -= amount_to_copy;
(first->flags & VLIB_BUFFER_NEXT_PRESENT));
}
+/**
+ * @brief linearize buffer chain - the first buffer is filled, if needed,
+ * buffers are allocated and filled, returns free space in last buffer or
+ * negative on failure
+ *
+ * @param[in] vm - vlib_main
+ * @param[in,out] first - first buffer in chain
+ */
+always_inline int
+vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * first)
+{
+ vlib_buffer_t *b = first;
+ vlib_buffer_free_list_t *fl =
+ vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+ u32 buf_len = fl->n_data_bytes;
+ // free buffer chain starting from the second buffer
+ int free_count = (b->flags & VLIB_BUFFER_NEXT_PRESENT) != 0;
+ u32 chain_to_free = b->next_buffer;
+
+ u32 len = vlib_buffer_length_in_chain (vm, b);
+ u32 free_len = buf_len - b->current_data - b->current_length;
+ int alloc_len = clib_max (len - free_len, 0); //use the free len in the first buffer
+ int n_buffers = (alloc_len + buf_len - 1) / buf_len;
+ u32 new_buffers[n_buffers];
+
+ u32 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
+ if (n_alloc != n_buffers)
+ {
+ vlib_buffer_free_no_next (vm, new_buffers, n_alloc);
+ return -1;
+ }
+
+ vlib_buffer_t *s = b;
+ while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ s = vlib_get_buffer (vm, s->next_buffer);
+ int d_free_len = buf_len - b->current_data - b->current_length;
+ ASSERT (d_free_len >= 0);
+ // chain buf and split write
+ u32 copy_len = clib_min (d_free_len, s->current_length);
+ u8 *d = vlib_buffer_put_uninit (b, copy_len);
+ clib_memcpy (d, vlib_buffer_get_current (s), copy_len);
+ int rest = s->current_length - copy_len;
+ if (rest > 0)
+ {
+ //prev buf is full
+ ASSERT (vlib_buffer_get_tail (b) == b->data + buf_len);
+ ASSERT (n_buffers > 0);
+ b = vlib_buffer_chain_buffer (vm, b, new_buffers[--n_buffers]);
+ //make full use of the new buffers
+ b->current_data = 0;
+ d = vlib_buffer_put_uninit (b, rest);
+ clib_memcpy (d, vlib_buffer_get_current (s) + copy_len, rest);
+ }
+ }
+ vlib_buffer_free (vm, &chain_to_free, free_count);
+ b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ if (b == first) /* no buffers addeed */
+ b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ ASSERT (len == vlib_buffer_length_in_chain (vm, first));
+ ASSERT (n_buffers == 0);
+ return buf_len - b->current_data - b->current_length;
+}
+
#endif /* included_vlib_buffer_funcs_h */
/*
Code Review / vpp.git / blobdiff