} while (0)
#endif
-#if DPDK == 0
-
typedef enum
{
/* Index is unknown. */
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-#endif /* DPDK == 0 */
-
clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
unsigned socket_id);
@return - (u32) number of buffers actually allocated, may be
less than the number requested or zero
*/
-u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers);
+always_inline u32
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ ASSERT (bm->cb.vlib_buffer_alloc_cb);
+
+ return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
+}
always_inline u32
vlib_buffer_round_size (u32 size)
@return - (u32) number of buffers actually allocated, may be
less than the number requested or zero
*/
-u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index);
+always_inline u32
+vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
+ u32 * buffers,
+ u32 n_buffers, u32 free_list_index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+
+ return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
+ free_list_index);
+}
/** \brief Free buffers
Frees the entire buffer chain for each buffer
@param n_buffers - (u32) number of buffers to free
*/
-void vlib_buffer_free (vlib_main_t * vm,
- /* pointer to first buffer */
- u32 * buffers,
- /* number of buffers to free */
- u32 n_buffers);
+always_inline void
+vlib_buffer_free (vlib_main_t * vm,
+ /* pointer to first buffer */
+ u32 * buffers,
+ /* number of buffers to free */
+ u32 n_buffers)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ ASSERT (bm->cb.vlib_buffer_free_cb);
+
+ return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
+}
/** \brief Free buffers, does not free the buffer chain for each buffer
@param n_buffers - (u32) number of buffers to free
*/
-void vlib_buffer_free_no_next (vlib_main_t * vm,
- /* pointer to first buffer */
- u32 * buffers,
- /* number of buffers to free */
- u32 n_buffers);
+always_inline void
+vlib_buffer_free_no_next (vlib_main_t * vm,
+ /* pointer to first buffer */
+ u32 * buffers,
+ /* number of buffers to free */
+ u32 n_buffers)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ ASSERT (bm->cb.vlib_buffer_free_no_next_cb);
+
+ return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
+}
/** \brief Free one buffer
Shorthand to free a single buffer chain.
/* Add/delete buffer free lists. */
u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
char *fmt, ...);
-void vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index);
+always_inline void
+vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ ASSERT (bm->cb.vlib_buffer_delete_free_list_cb);
+
+ bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
+}
/* Find already existing public free list with given size or create one. */
u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
char *fmt, ...);
+/* Merge two free lists */
+void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
+ vlib_buffer_free_list_t * src);
+
+/* Make sure we have at least given number of unaligned buffers. */
+void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
+ vlib_buffer_free_list_t *
+ free_list,
+ uword n_unaligned_buffers);
+
+always_inline u32
+vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+
+ size = vlib_buffer_round_size (size);
+ uword *p = hash_get (bm->free_list_by_size, size);
+ return p ? p[0] : ~0;
+}
+
+always_inline vlib_buffer_free_list_t *
+vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b,
+ u32 * index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ u32 i;
+
+ *index = i = b->free_list_index;
+ return pool_elt_at_index (bm->buffer_free_list_pool, i);
+}
+
always_inline vlib_buffer_free_list_t *
vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
{
vec_validate (new_buffers, n_buffers - 1);
n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
- ASSERT (n_alloc == n_buffers);
+
+ /* No guarantee that we'll get all the buffers we asked for */
+ if (PREDICT_FALSE (n_alloc < n_buffers))
+ {
+ if (n_alloc > 0)
+ vlib_buffer_free (vm, new_buffers, n_alloc);
+ vec_free (new_buffers);
+ return 0;
+ }
/* 1st segment */
s = b;
d->flags = s->flags & flag_mask;
}
+ vec_free (new_buffers);
return fd;
}
-/*
- * vlib_buffer_chain_* functions provide a way to create long buffers.
- * When DPDK is enabled, the 'hidden' DPDK header is taken care of transparently.
- */
-
/* Initializes the buffer as an empty packet with no chained buffers. */
always_inline void
vlib_buffer_chain_init (vlib_buffer_t * first)
/* Vector of packet data. */
u8 *packet_data;
- /* Note: the next three fields are unused if DPDK == 1 */
-
/* Number of buffers to allocate in each call to physmem
allocator. */
u32 min_n_buffers_each_physmem_alloc;
return n;
}
-typedef union
-{
- vlib_buffer_t b;
- vlib_copy_unit_t i[sizeof (vlib_buffer_t) / sizeof (vlib_copy_unit_t)];
-}
-vlib_buffer_union_t;
-
/* Set a buffer quickly into "uninitialized" state. We want this to
be extremely cheap and arrange for all fields that need to be
initialized to be in the first 128 bits of the buffer. */
always_inline void
-vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
+vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst = (vlib_buffer_union_t *) _dst;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure vlib_buffer_t is cacheline aligned and sized */
ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
/* Make sure buffer template is sane. */
ASSERT (fl->index == fl->buffer_init_template.free_list_index);
- /* Copy template from src->current_data thru src->free_list_index */
- dst->i[0] = src->i[0];
- if (1 * sizeof (dst->i[0]) < 16)
- dst->i[1] = src->i[1];
- if (2 * sizeof (dst->i[0]) < 16)
- dst->i[2] = src->i[2];
-
/* Make sure it really worked. */
-#define _(f) ASSERT (dst->b.f == src->b.f)
+#define _(f) dst->f = src->f
_(current_data);
_(current_length);
_(flags);
_(free_list_index);
#undef _
- ASSERT (dst->b.total_length_not_including_first_buffer == 0);
+ ASSERT (dst->total_length_not_including_first_buffer == 0);
}
always_inline void
-vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
- vlib_buffer_t * _dst1,
+vlib_buffer_add_to_free_list (vlib_main_t * vm,
+ vlib_buffer_free_list_t * f,
+ u32 buffer_index, u8 do_init)
+{
+ vlib_buffer_t *b;
+ b = vlib_get_buffer (vm, buffer_index);
+ if (PREDICT_TRUE (do_init))
+ vlib_buffer_init_for_free_list (b, f);
+ vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+}
+
+always_inline void
+vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0,
+ vlib_buffer_t * dst1,
vlib_buffer_free_list_t * fl)
{
- vlib_buffer_union_t *dst0 = (vlib_buffer_union_t *) _dst0;
- vlib_buffer_union_t *dst1 = (vlib_buffer_union_t *) _dst1;
- vlib_buffer_union_t *src =
- (vlib_buffer_union_t *) & fl->buffer_init_template;
+ vlib_buffer_t *src = &fl->buffer_init_template;
/* Make sure buffer template is sane. */
ASSERT (fl->index == fl->buffer_init_template.free_list_index);
- /* Copy template from src->current_data thru src->free_list_index */
- dst0->i[0] = dst1->i[0] = src->i[0];
- if (1 * sizeof (dst0->i[0]) < 16)
- dst0->i[1] = dst1->i[1] = src->i[1];
- if (2 * sizeof (dst0->i[0]) < 16)
- dst0->i[2] = dst1->i[2] = src->i[2];
-
/* Make sure it really worked. */
-#define _(f) ASSERT (dst0->b.f == src->b.f && dst1->b.f == src->b.f)
+#define _(f) dst0->f = src->f; dst1->f = src->f
_(current_data);
_(current_length);
_(flags);
_(free_list_index);
#undef _
- ASSERT (dst0->b.total_length_not_including_first_buffer == 0);
- ASSERT (dst1->b.total_length_not_including_first_buffer == 0);
+ ASSERT (dst0->total_length_not_including_first_buffer == 0);
+ ASSERT (dst1->total_length_not_including_first_buffer == 0);
}
#if CLIB_DEBUG > 0
Code Review / vpp.git / blobdiff