#define included_vlib_buffer_funcs_h
#include <vppinfra/hash.h>
+#include <vppinfra/fifo.h>
+#include <vlib/buffer.h>
+#include <vlib/physmem_funcs.h>
+#include <vlib/main.h>
+#include <vlib/node.h>
/** \file
vlib buffer access methods.
*/
+typedef void (vlib_buffer_enqueue_to_next_fn_t) (vlib_main_t *vm,
+ vlib_node_runtime_t *node,
+ u32 *buffers, u16 *nexts,
+ uword count);
+typedef void (vlib_buffer_enqueue_to_single_next_fn_t) (
+ vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index,
+ u32 count);
+
+typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) (
+ vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index,
+ u32 *buffer_indices, u16 *thread_indices, u32 n_packets,
+ int drop_on_congestion);
+
+typedef u32 (vlib_frame_queue_dequeue_fn_t) (vlib_main_t *vm,
+ vlib_frame_queue_main_t *fqm);
+
+typedef struct
+{
+ vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn;
+ vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn;
+ vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn;
+ vlib_frame_queue_dequeue_fn_t *frame_queue_dequeue_fn;
+} vlib_buffer_func_main_t;
+
+extern vlib_buffer_func_main_t vlib_buffer_func_main;
+
+always_inline void
+vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_pool_t *bp;
+
+ /* reference count in allocated buffer always must be 1 or higher */
+ ASSERT (b->ref_count > 0);
+
+ /* verify that buffer pool index is valid */
+ bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index);
+ ASSERT (pointer_to_uword (b) >= bp->start);
+ ASSERT (pointer_to_uword (b) < bp->start + bp->size -
+ (bp->data_size + sizeof (vlib_buffer_t)));
+}
+
+always_inline void *
+vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
+ uword offset)
+{
+ offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+ return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
+}
/** \brief Translate buffer index into buffer pointer
always_inline vlib_buffer_t *
vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
{
- return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
- << CLIB_LOG2_CACHE_LINE_BYTES);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_t *b;
+
+ b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
+ vlib_buffer_validate (vm, b);
+ return b;
+}
+
+static_always_inline u32
+vlib_buffer_get_default_data_size (vlib_main_t * vm)
+{
+ return vm->buffer_main->default_data_size;
+}
+
+static_always_inline void
+vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
+{
+ clib_memcpy_u32 (dst, src, n_indices);
+}
+
+always_inline void
+vlib_buffer_copy_indices_from_ring (u32 * dst, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers)
+{
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ {
+ vlib_buffer_copy_indices (dst, ring + start, n_buffers);
+ }
+ else
+ {
+ u32 n = ring_size - start;
+ vlib_buffer_copy_indices (dst, ring + start, n);
+ vlib_buffer_copy_indices (dst + n, ring, n_buffers - n);
+ }
+}
+
+always_inline void
+vlib_buffer_copy_indices_to_ring (u32 * ring, u32 * src, u32 start,
+ u32 ring_size, u32 n_buffers)
+{
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ {
+ vlib_buffer_copy_indices (ring + start, src, n_buffers);
+ }
+ else
+ {
+ u32 n = ring_size - start;
+ vlib_buffer_copy_indices (ring + start, src, n);
+ vlib_buffer_copy_indices (ring, src + n, n_buffers - n);
+ }
+}
+
+STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
+static_always_inline void
+vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
+{
+#if defined CLIB_HAVE_VEC512
+ b->as_u8x64[0] = bt->as_u8x64[0];
+#elif defined (CLIB_HAVE_VEC256)
+ b->as_u8x32[0] = bt->as_u8x32[0];
+ b->as_u8x32[1] = bt->as_u8x32[1];
+#elif defined (CLIB_HAVE_VEC128)
+ b->as_u8x16[0] = bt->as_u8x16[0];
+ b->as_u8x16[1] = bt->as_u8x16[1];
+ b->as_u8x16[2] = bt->as_u8x16[2];
+ b->as_u8x16[3] = bt->as_u8x16[3];
+#else
+ clib_memcpy_fast (b, bt, 64);
+#endif
+}
+
+always_inline u8
+vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node)
+{
+ ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
+ return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
+}
+
+/** \brief Translate array of buffer indices into buffer pointers with offset
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param bi - (u32 *) array of buffer indices
+ @param b - (void **) array to store buffer pointers
+ @param count - (uword) number of elements
+ @param offset - (i32) offset applied to each pointer
+*/
+static_always_inline void
+vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
+ i32 offset)
+{
+ uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
+#ifdef CLIB_HAVE_VEC512
+ u64x8 of8 = u64x8_splat (buffer_mem_start + offset);
+ u64x4 off = u64x8_extract_lo (of8);
+ /* if count is not const, compiler will not unroll while loop
+ se we maintain two-in-parallel variant */
+ while (count >= 32)
+ {
+ u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
+ u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8));
+ u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16));
+ u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24));
+ /* shift and add to get vlib_buffer_t pointer */
+ u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b);
+ u64x8_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 8);
+ u64x8_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 16);
+ u64x8_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 24);
+ b += 32;
+ bi += 32;
+ count -= 32;
+ }
+ while (count >= 8)
+ {
+ u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
+ /* shift and add to get vlib_buffer_t pointer */
+ u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b);
+ b += 8;
+ bi += 8;
+ count -= 8;
+ }
+#elif defined CLIB_HAVE_VEC256
+ u64x4 off = u64x4_splat (buffer_mem_start + offset);
+ /* if count is not const, compiler will not unroll while loop
+ se we maintain two-in-parallel variant */
+ while (count >= 32)
+ {
+ u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
+ u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4));
+ u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8));
+ u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12));
+ u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16));
+ u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20));
+ u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24));
+ u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28));
+ /* shift and add to get vlib_buffer_t pointer */
+ u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
+ u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
+ u64x4_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 8);
+ u64x4_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 12);
+ u64x4_store_unaligned ((b4 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 16);
+ u64x4_store_unaligned ((b5 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 20);
+ u64x4_store_unaligned ((b6 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 24);
+ u64x4_store_unaligned ((b7 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 28);
+ b += 32;
+ bi += 32;
+ count -= 32;
+ }
+#endif
+ while (count >= 4)
+ {
+#ifdef CLIB_HAVE_VEC256
+ u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
+ /* shift and add to get vlib_buffer_t pointer */
+ u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
+#elif defined (CLIB_HAVE_VEC128)
+ u64x2 off = u64x2_splat (buffer_mem_start + offset);
+ u32x4 bi4 = u32x4_load_unaligned (bi);
+ u64x2 b0 = u64x2_from_u32x4 ((u32x4) bi4);
+#if defined (__aarch64__)
+ u64x2 b1 = u64x2_from_u32x4_high ((u32x4) bi4);
+#else
+ bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
+ u64x2 b1 = u64x2_from_u32x4 ((u32x4) bi4);
+#endif
+ u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
+ u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
+#else
+ b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
+ b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
+ b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
+ b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
+#endif
+ b += 4;
+ bi += 4;
+ count -= 4;
+ }
+ while (count)
+ {
+ b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
+ b += 1;
+ bi += 1;
+ count -= 1;
+ }
+}
+
+/** \brief Translate array of buffer indices into buffer pointers
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param bi - (u32 *) array of buffer indices
+ @param b - (vlib_buffer_t **) array to store buffer pointers
+ @param count - (uword) number of elements
+*/
+
+static_always_inline void
+vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
+{
+ vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
}
/** \brief Translate buffer pointer into buffer index
@param p - (void *) buffer pointer
@return - (u32) buffer index
*/
+
always_inline u32
vlib_get_buffer_index (vlib_main_t * vm, void *p)
{
- uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+ ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+ ASSERT (offset < bm->buffer_mem_size);
ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
}
+/** \brief Translate array of buffer pointers into buffer indices with offset
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param b - (void **) array of buffer pointers
+ @param bi - (u32 *) array to store buffer indices
+ @param count - (uword) number of elements
+ @param offset - (i32) offset applied to each pointer
+*/
+static_always_inline void
+vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
+ uword count, i32 offset)
+{
+#ifdef CLIB_HAVE_VEC256
+ u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
+ u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
+
+ while (count >= 8)
+ {
+ /* load 4 pointers into 256-bit register */
+ u64x4 v0 = u64x4_load_unaligned (b);
+ u64x4 v1 = u64x4_load_unaligned (b + 4);
+ u32x8 v2, v3;
+
+ v0 -= off4;
+ v1 -= off4;
+
+ v0 >>= CLIB_LOG2_CACHE_LINE_BYTES;
+ v1 >>= CLIB_LOG2_CACHE_LINE_BYTES;
+
+ /* permute 256-bit register so lower u32s of each buffer index are
+ * placed into lower 128-bits */
+ v2 = u32x8_permute ((u32x8) v0, mask);
+ v3 = u32x8_permute ((u32x8) v1, mask);
+
+ /* extract lower 128-bits and save them to the array of buffer indices */
+ u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
+ u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
+ bi += 8;
+ b += 8;
+ count -= 8;
+ }
+#endif
+ while (count >= 4)
+ {
+ /* equivalent non-nector implementation */
+ bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
+ bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
+ bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
+ bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
+ bi += 4;
+ b += 4;
+ count -= 4;
+ }
+ while (count)
+ {
+ bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
+ bi += 1;
+ b += 1;
+ count -= 1;
+ }
+}
+
+/** \brief Translate array of buffer pointers into buffer indices
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param b - (vlib_buffer_t **) array of buffer pointers
+ @param bi - (u32 *) array to store buffer indices
+ @param count - (uword) number of elements
+*/
+static_always_inline void
+vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi,
+ uword count)
+{
+ vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
+}
+
/** \brief Get next buffer in buffer linklist, or zero for end of list.
@param vm - (vlib_main_t *) vlib main data structure pointer
always_inline uword
vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
{
- uword l = b->current_length + b->total_length_not_including_first_buffer;
- if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
- | VLIB_BUFFER_TOTAL_LENGTH_VALID))
- == VLIB_BUFFER_NEXT_PRESENT))
- return vlib_buffer_length_in_chain_slow_path (vm, b);
- return l;
+ uword len = b->current_length;
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return len;
+
+ if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
+ return len + b->total_length_not_including_first_buffer;
+
+ return vlib_buffer_length_in_chain_slow_path (vm, b);
}
/** \brief Get length in bytes of the buffer index buffer chain
{
b = vlib_get_buffer (vm, buffer_index);
l = b->current_length;
- clib_memcpy (contents + content_len, b->data + b->current_data, l);
+ clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
content_len += l;
if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
break;
return content_len;
}
-/* Return physical address of buffer->data start. */
-always_inline u64
-vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
+always_inline uword
+vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
{
- return vlib_physmem_offset_to_physical (&vm->physmem_main,
- (((uword) buffer_index) <<
- CLIB_LOG2_CACHE_LINE_BYTES) +
- STRUCT_OFFSET_OF (vlib_buffer_t,
- data));
+ return vlib_physmem_get_pa (vm, b->data);
+}
+
+always_inline uword
+vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
+{
+ return vlib_buffer_get_pa (vm, b) + b->current_data;
}
/** \brief Prefetch buffer metadata by buffer index
vlib_prefetch_buffer_header (_b, type); \
} while (0)
-#if 0
-/* Iterate over known allocated vlib bufs. You probably do not want
- * to do this!
- @param vm the vlib_main_t
- @param bi found allocated buffer index
- @param body operation to perform on buffer index
- function executes body for each allocated buffer index
- */
-#define vlib_buffer_foreach_allocated(vm,bi,body) \
-do { \
- vlib_main_t * _vmain = (vm); \
- vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
- hash_pair_t * _vbpair; \
- hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
- if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
- (bi) = _vbpair->key; \
- body; \
- } \
- })); \
-} while (0)
-#endif
-
typedef enum
{
/* Index is unknown. */
VLIB_BUFFER_KNOWN_ALLOCATED,
} vlib_buffer_known_state_t;
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+ uword n_buffers,
+ vlib_buffer_known_state_t
+ expected_state);
+
always_inline vlib_buffer_known_state_t
vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
+ clib_spinlock_lock (&bm->buffer_known_hash_lockp);
uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+ clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
return p ? p[0] : VLIB_BUFFER_UNKNOWN;
}
-always_inline void
-vlib_buffer_set_known_state (vlib_main_t * vm,
- u32 buffer_index,
- vlib_buffer_known_state_t state)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- ASSERT (os_get_cpu_number () == 0);
- hash_set (bm->buffer_known_hash, buffer_index, state);
-}
-
/* Validates sanity of a single buffer.
Returns format'ed vector with error message if any. */
u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
uword follow_chain);
-clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id);
+u8 *vlib_validate_buffers (vlib_main_t * vm,
+ u32 * buffers,
+ uword next_buffer_stride,
+ uword n_buffers,
+ vlib_buffer_known_state_t known_state,
+ uword follow_buffer_next);
-/** \brief Allocate buffers into supplied array
+static_always_inline vlib_buffer_pool_t *
+vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
+}
+
+static_always_inline __clib_warn_unused_result uword
+vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
+ u32 n_buffers)
+{
+ vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
+ u32 len;
+
+ ASSERT (bp->buffers);
+
+ clib_spinlock_lock (&bp->lock);
+ len = bp->n_avail;
+ if (PREDICT_TRUE (n_buffers < len))
+ {
+ len -= n_buffers;
+ vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
+ bp->n_avail = len;
+ clib_spinlock_unlock (&bp->lock);
+ return n_buffers;
+ }
+ else
+ {
+ vlib_buffer_copy_indices (buffers, bp->buffers, len);
+ bp->n_avail = 0;
+ clib_spinlock_unlock (&bp->lock);
+ return len;
+ }
+}
+
+
+/** \brief Allocate buffers from specific pool into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param buffers - (u32 * ) buffer index array
@return - (u32) number of buffers actually allocated, may be
less than the number requested or zero
*/
-always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+
+always_inline __clib_warn_unused_result u32
+vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
+ u8 buffer_pool_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_pool_t *bp;
+ vlib_buffer_pool_thread_t *bpt;
+ u32 *src, *dst, len, n_left;
+
+ /* If buffer allocation fault injection is configured */
+ if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
+ {
+ u32 vlib_buffer_alloc_may_fail (vlib_main_t *, u32);
+
+ /* See how many buffers we're willing to allocate */
+ n_buffers = vlib_buffer_alloc_may_fail (vm, n_buffers);
+ if (n_buffers == 0)
+ return (n_buffers);
+ }
- ASSERT (bm->cb.vlib_buffer_alloc_cb);
+ bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
+ bpt = vec_elt_at_index (bp->threads, vm->thread_index);
- return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
+ dst = buffers;
+ n_left = n_buffers;
+ len = bpt->n_cached;
+
+ /* per-thread cache contains enough buffers */
+ if (len >= n_buffers)
+ {
+ src = bpt->cached_buffers + len - n_buffers;
+ vlib_buffer_copy_indices (dst, src, n_buffers);
+ bpt->n_cached -= n_buffers;
+ goto done;
+ }
+
+ /* alloc bigger than cache - take buffers directly from main pool */
+ if (n_buffers >= VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ)
+ {
+ n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers,
+ n_buffers);
+ goto done;
+ }
+
+ /* take everything available in the cache */
+ if (len)
+ {
+ vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
+ bpt->n_cached = 0;
+ dst += len;
+ n_left -= len;
+ }
+
+ len = round_pow2 (n_left, 32);
+ len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
+ len);
+ bpt->n_cached = len;
+
+ if (len)
+ {
+ u32 n_copy = clib_min (len, n_left);
+ src = bpt->cached_buffers + len - n_copy;
+ vlib_buffer_copy_indices (dst, src, n_copy);
+ bpt->n_cached -= n_copy;
+ n_left -= n_copy;
+ }
+
+ n_buffers -= n_left;
+
+done:
+ /* Verify that buffers are known free. */
+ if (CLIB_DEBUG > 0)
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_FREE);
+ if (PREDICT_FALSE (bm->alloc_callback_fn != 0))
+ bm->alloc_callback_fn (vm, buffer_pool_index, buffers, n_buffers);
+ return n_buffers;
}
-always_inline u32
-vlib_buffer_round_size (u32 size)
+/** \brief Allocate buffers from specific numa node into supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u32) number of buffers requested
+ @param numa_node - (u32) numa node
+ @return - (u32) number of buffers actually allocated, may be
+ less than the number requested or zero
+*/
+always_inline __clib_warn_unused_result u32
+vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
+ u32 numa_node)
{
- return round_pow2 (size, sizeof (vlib_buffer_t));
+ u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
+ return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
}
-/** \brief Allocate buffers from specific freelist into supplied array
+/** \brief Allocate buffers into supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param buffers - (u32 * ) buffer index array
@return - (u32) number of buffers actually allocated, may be
less than the number requested or zero
*/
-always_inline u32
-vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 free_list_index)
+
+always_inline __clib_warn_unused_result u32
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+ return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
+}
+
+/** \brief Allocate buffers into ring
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index ring
+ @param start - (u32) first slot in the ring
+ @param ring_size - (u32) ring size
+ @param n_buffers - (u32) number of buffers requested
+ @return - (u32) number of buffers actually allocated, may be
+ less than the number requested or zero
+*/
+always_inline __clib_warn_unused_result u32
+vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers)
+{
+ u32 n_alloc;
+
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ return vlib_buffer_alloc (vm, ring + start, n_buffers);
+
+ n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
+
+ if (PREDICT_TRUE (n_alloc == ring_size - start))
+ n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
+
+ return n_alloc;
+}
+
+/** \brief Allocate buffers into ring from specific buffer pool
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index ring
+ @param start - (u32) first slot in the ring
+ @param ring_size - (u32) ring size
+ @param n_buffers - (u32) number of buffers requested
+ @return - (u32) number of buffers actually allocated, may be
+ less than the number requested or zero
+*/
+always_inline __clib_warn_unused_result u32
+vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers,
+ u8 buffer_pool_index)
+{
+ u32 n_alloc;
+
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
+ buffer_pool_index);
+
+ n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
+ buffer_pool_index);
+
+ if (PREDICT_TRUE (n_alloc == ring_size - start))
+ n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
+ buffer_pool_index);
+
+ return n_alloc;
+}
+
+static_always_inline void
+vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
+ u32 * buffers, u32 n_buffers)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
+ vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads,
+ vm->thread_index);
+ u32 n_cached, n_empty;
+
+ if (CLIB_DEBUG > 0)
+ vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+ VLIB_BUFFER_KNOWN_ALLOCATED);
+ if (PREDICT_FALSE (bm->free_callback_fn != 0))
+ bm->free_callback_fn (vm, buffer_pool_index, buffers, n_buffers);
+
+ n_cached = bpt->n_cached;
+ n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached;
+ if (n_buffers <= n_empty)
+ {
+ vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
+ buffers, n_buffers);
+ bpt->n_cached = n_cached + n_buffers;
+ return;
+ }
+
+ vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
+ buffers + n_buffers - n_empty, n_empty);
+ bpt->n_cached = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ;
+
+ clib_spinlock_lock (&bp->lock);
+ vlib_buffer_copy_indices (bp->buffers + bp->n_avail, buffers,
+ n_buffers - n_empty);
+ bp->n_avail += n_buffers - n_empty;
+ clib_spinlock_unlock (&bp->lock);
+}
- ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+static_always_inline void
+vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
+ int maybe_next)
+{
+ const int queue_size = 128;
+ vlib_buffer_pool_t *bp = 0;
+ u8 buffer_pool_index = ~0;
+ u32 n_queue = 0, queue[queue_size + 4];
+ vlib_buffer_t bt = { };
+#if defined(CLIB_HAVE_VEC128)
+ vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
+ vlib_buffer_t bpi_vec = {};
+ vlib_buffer_t flags_refs_mask = {
+ .flags = VLIB_BUFFER_NEXT_PRESENT,
+ .ref_count = ~1
+ };
+#endif
+
+ if (PREDICT_FALSE (n_buffers == 0))
+ return;
+
+ vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]);
+ buffer_pool_index = b->buffer_pool_index;
+ bp = vlib_get_buffer_pool (vm, buffer_pool_index);
+ vlib_buffer_copy_template (&bt, &bp->buffer_template);
+#if defined(CLIB_HAVE_VEC128)
+ bpi_vec.buffer_pool_index = buffer_pool_index;
+#endif
+
+ while (n_buffers)
+ {
+ vlib_buffer_t *b[8];
+ u32 bi, sum = 0, flags, next;
+
+ if (n_buffers < 4)
+ goto one_by_one;
+
+ vlib_get_buffers (vm, buffers, b, 4);
+
+ if (n_buffers >= 12)
+ {
+ vlib_get_buffers (vm, buffers + 8, b + 4, 4);
+ vlib_prefetch_buffer_header (b[4], LOAD);
+ vlib_prefetch_buffer_header (b[5], LOAD);
+ vlib_prefetch_buffer_header (b[6], LOAD);
+ vlib_prefetch_buffer_header (b[7], LOAD);
+ }
+
+#if defined(CLIB_HAVE_VEC128)
+ u8x16 p0, p1, p2, p3, r;
+ p0 = u8x16_load_unaligned (b[0]);
+ p1 = u8x16_load_unaligned (b[1]);
+ p2 = u8x16_load_unaligned (b[2]);
+ p3 = u8x16_load_unaligned (b[3]);
+
+ r = p0 ^ bpi_vec.as_u8x16[0];
+ r |= p1 ^ bpi_vec.as_u8x16[0];
+ r |= p2 ^ bpi_vec.as_u8x16[0];
+ r |= p3 ^ bpi_vec.as_u8x16[0];
+ r &= bpi_mask.as_u8x16[0];
+ r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
+
+ sum = !u8x16_is_all_zero (r);
+#else
+ sum |= b[0]->flags;
+ sum |= b[1]->flags;
+ sum |= b[2]->flags;
+ sum |= b[3]->flags;
+ sum &= VLIB_BUFFER_NEXT_PRESENT;
+ sum += b[0]->ref_count - 1;
+ sum += b[1]->ref_count - 1;
+ sum += b[2]->ref_count - 1;
+ sum += b[3]->ref_count - 1;
+ sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
+ sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
+ sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
+ sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
+#endif
+
+ if (sum)
+ goto one_by_one;
+
+ vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
+ vlib_buffer_copy_template (b[0], &bt);
+ vlib_buffer_copy_template (b[1], &bt);
+ vlib_buffer_copy_template (b[2], &bt);
+ vlib_buffer_copy_template (b[3], &bt);
+ n_queue += 4;
+
+ vlib_buffer_validate (vm, b[0]);
+ vlib_buffer_validate (vm, b[1]);
+ vlib_buffer_validate (vm, b[2]);
+ vlib_buffer_validate (vm, b[3]);
+
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
+
+ if (n_queue >= queue_size)
+ {
+ vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
+ n_queue = 0;
+ }
+ buffers += 4;
+ n_buffers -= 4;
+ continue;
+
+ one_by_one:
+ bi = buffers[0];
+
+ next_in_chain:
+ b[0] = vlib_get_buffer (vm, bi);
+ flags = b[0]->flags;
+ next = b[0]->next_buffer;
+
+ if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
+ {
+
+ if (n_queue)
+ {
+ vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
+ n_queue = 0;
+ }
- return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
- free_list_index);
+ buffer_pool_index = b[0]->buffer_pool_index;
+#if defined(CLIB_HAVE_VEC128)
+ bpi_vec.buffer_pool_index = buffer_pool_index;
+#endif
+ bp = vlib_get_buffer_pool (vm, buffer_pool_index);
+ vlib_buffer_copy_template (&bt, &bp->buffer_template);
+ }
+
+ vlib_buffer_validate (vm, b[0]);
+
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
+
+ if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
+ {
+ vlib_buffer_copy_template (b[0], &bt);
+ queue[n_queue++] = bi;
+ }
+
+ if (n_queue == queue_size)
+ {
+ vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
+ n_queue = 0;
+ }
+
+ if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
+ {
+ bi = next;
+ goto next_in_chain;
+ }
+
+ buffers++;
+ n_buffers--;
+ }
+
+ if (n_queue)
+ vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
}
+
/** \brief Free buffers
Frees the entire buffer chain for each buffer
/* number of buffers to free */
u32 n_buffers)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_free_cb);
-
- return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
+ vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
}
/** \brief Free buffers, does not free the buffer chain for each buffer
/* number of buffers to free */
u32 n_buffers)
{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_free_no_next_cb);
-
- return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
+ vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
}
/** \brief Free one buffer
always_inline void
vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
{
- vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
-}
-
-/* Add/delete buffer free lists. */
-u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
- char *fmt, ...);
-always_inline void
-vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- ASSERT (bm->cb.vlib_buffer_delete_free_list_cb);
-
- bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
-}
-
-/* Find already existing public free list with given size or create one. */
-u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
- char *fmt, ...);
-
-/* Merge two free lists */
-void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
- vlib_buffer_free_list_t * src);
-
-/* Make sure we have at least given number of unaligned buffers. */
-void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
- vlib_buffer_free_list_t *
- free_list,
- uword n_unaligned_buffers);
-
-always_inline u32
-vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- size = vlib_buffer_round_size (size);
- uword *p = hash_get (bm->free_list_by_size, size);
- return p ? p[0] : ~0;
-}
-
-always_inline vlib_buffer_free_list_t *
-vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b,
- u32 * index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- u32 i;
-
- *index = i = b->free_list_index;
- return pool_elt_at_index (bm->buffer_free_list_pool, i);
-}
-
-always_inline vlib_buffer_free_list_t *
-vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
-
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
-
- /* Sanity: indices must match. */
- ASSERT (f->index == free_list_index);
-
- return f;
-}
-
-always_inline u32
-vlib_buffer_free_list_buffer_size (vlib_main_t * vm, u32 free_list_index)
-{
- vlib_buffer_free_list_t *f =
- vlib_buffer_get_free_list (vm, free_list_index);
- return f->n_data_bytes;
+ vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
}
-void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
+/** \brief Free buffers from ring
-/* Reasonably fast buffer copy routine. */
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index ring
+ @param start - (u32) first slot in the ring
+ @param ring_size - (u32) ring size
+ @param n_buffers - (u32) number of buffers
+*/
always_inline void
-vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
+vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers)
{
- while (n >= 4)
+ ASSERT (n_buffers <= ring_size);
+
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
{
- dst[0] = src[0];
- dst[1] = src[1];
- dst[2] = src[2];
- dst[3] = src[3];
- dst += 4;
- src += 4;
- n -= 4;
+ vlib_buffer_free (vm, ring + start, n_buffers);
}
- while (n > 0)
+ else
{
- dst[0] = src[0];
- dst += 1;
- src += 1;
- n -= 1;
+ vlib_buffer_free (vm, ring + start, ring_size - start);
+ vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
}
}
-always_inline void *
-vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
- uword n_bytes, uword alignment)
-{
- void *r =
- vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
- if (!r)
- *error =
- clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
- n_bytes);
- else
- *error = 0;
- return r;
-}
-
-/* By default allocate I/O memory with cache line alignment. */
-always_inline void *
-vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
-{
- return vlib_physmem_alloc_aligned (vm, error, n_bytes,
- CLIB_CACHE_LINE_BYTES);
-}
+/** \brief Free buffers from ring without freeing tail buffers
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index ring
+ @param start - (u32) first slot in the ring
+ @param ring_size - (u32) ring size
+ @param n_buffers - (u32) number of buffers
+*/
always_inline void
-vlib_physmem_free (vlib_main_t * vm, void *mem)
+vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
+ u32 ring_size, u32 n_buffers)
{
- return vm->os_physmem_free (mem);
-}
+ ASSERT (n_buffers <= ring_size);
-always_inline u64
-vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem)
-{
- vlib_physmem_main_t *pm = &vm->physmem_main;
- uword o = pointer_to_uword (mem) - pm->virtual.start;
- return vlib_physmem_offset_to_physical (pm, o);
+ if (PREDICT_TRUE (start + n_buffers <= ring_size))
+ {
+ vlib_buffer_free_no_next (vm, ring + start, n_buffers);
+ }
+ else
+ {
+ vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
+ vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
+ }
}
/* Append given data to end of buffer, possibly allocating new buffers. */
-u32 vlib_buffer_add_data (vlib_main_t * vm,
- u32 free_list_index,
- u32 buffer_index, void *data, u32 n_data_bytes);
+int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
+ u32 n_data_bytes);
+
+/* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
+#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \
+ (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \
+ VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
/* duplicate all buffers in chain */
always_inline vlib_buffer_t *
{
vlib_buffer_t *s, *d, *fd;
uword n_alloc, n_buffers = 1;
- u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK;
int i;
s = b;
d->current_data = s->current_data;
d->current_length = s->current_length;
d->flags = s->flags & flag_mask;
+ d->trace_handle = s->trace_handle;
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
/* next segments */
for (i = 1; i < n_buffers; i++)
d = vlib_get_buffer (vm, new_buffers[i]);
d->current_data = s->current_data;
d->current_length = s->current_length;
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
d->flags = s->flags & flag_mask;
}
return fd;
}
-/** \brief Create multiple clones of buffer and store them in the supplied array
+/* duplicate first buffer in chain */
+always_inline vlib_buffer_t *
+vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
+{
+ vlib_buffer_t *d;
+
+ if ((vlib_buffer_alloc (vm, di, 1)) != 1)
+ return 0;
+
+ d = vlib_get_buffer (vm, *di);
+ /* 1st segment */
+ d->current_data = b->current_data;
+ d->current_length = b->current_length;
+ clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
+ clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (b), b->current_length);
+
+ return d;
+}
+
+/* \brief Move packet from current position to offset position in buffer.
+ Only work for small packet using one buffer with room to fit the move
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param b - (vlib_buffer_t *) pointer to buffer
+ @param offset - (i16) position to move the packet in buffer
+ */
+always_inline void
+vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset)
+{
+ ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
+ ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
+ ASSERT (offset + b->current_length <
+ vlib_buffer_get_default_data_size (vm));
+
+ u8 *source = vlib_buffer_get_current (b);
+ b->current_data = offset;
+ u8 *destination = vlib_buffer_get_current (b);
+ u16 length = b->current_length;
+
+ if (source + length <= destination) /* no overlap */
+ clib_memcpy_fast (destination, source, length);
+ else
+ memmove (destination, source, length);
+}
+
+/** \brief Create a maximum of 256 clones of buffer and store them
+ in the supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param src_buffer - (u32) source buffer index
@param buffers - (u32 * ) buffer index array
- @param n_buffers - (u8) number of buffer clones requested
+ @param n_buffers - (u16) number of buffer clones requested (<=256)
@param head_end_offset - (u16) offset relative to current position
where packet head ends
- @return - (u8) number of buffers actually cloned, may be
+ @param offset - (i16) copy packet head at current position if 0,
+ else at offset position to change headroom space as specified
+ @return - (u16) number of buffers actually cloned, may be
less than the number requested or zero
*/
-
-always_inline u8
-vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
- u8 n_buffers, u16 head_end_offset)
+always_inline u16
+vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+ u16 n_buffers, u16 head_end_offset, i16 offset)
{
- u8 i;
+ u16 i;
vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
- ASSERT (s->n_add_refs == 0);
+ ASSERT (s->ref_count == 1);
ASSERT (n_buffers);
+ ASSERT (n_buffers <= 256);
+ ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
+ ASSERT ((offset + head_end_offset) <
+ vlib_buffer_get_default_data_size (vm));
if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
{
buffers[0] = src_buffer;
+ if (offset)
+ vlib_buffer_move (vm, s, offset);
+
for (i = 1; i < n_buffers; i++)
{
vlib_buffer_t *d;
return n_buffers;
}
- n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
- s->free_list_index);
- if (PREDICT_FALSE (n_buffers == 0))
+ if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
{
buffers[0] = src_buffer;
return 1;
}
+ n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
+ s->buffer_pool_index);
+
for (i = 0; i < n_buffers; i++)
{
vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
- d->current_data = s->current_data;
+ if (offset)
+ d->current_data = offset;
+ else
+ d->current_data = s->current_data;
+
d->current_length = head_end_offset;
- d->free_list_index = s->free_list_index;
- d->total_length_not_including_first_buffer =
- s->total_length_not_including_first_buffer + s->current_length -
+ ASSERT (d->buffer_pool_index == s->buffer_pool_index);
+
+ d->total_length_not_including_first_buffer = s->current_length -
head_end_offset;
- d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
- d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
- head_end_offset);
+ if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
+ {
+ d->total_length_not_including_first_buffer +=
+ s->total_length_not_including_first_buffer;
+ }
+ d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) |
+ VLIB_BUFFER_NEXT_PRESENT;
+ d->trace_handle = s->trace_handle;
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), head_end_offset);
d->next_buffer = src_buffer;
}
vlib_buffer_advance (s, head_end_offset);
- s->n_add_refs = n_buffers - 1;
+ s->ref_count = n_buffers ? n_buffers : s->ref_count;
while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
{
s = vlib_get_buffer (vm, s->next_buffer);
- s->n_add_refs = n_buffers - 1;
+ s->ref_count = n_buffers ? n_buffers : s->ref_count;
}
return n_buffers;
}
+/** \brief Create multiple clones of buffer and store them
+ in the supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param src_buffer - (u32) source buffer index
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u16) number of buffer clones requested (<=256)
+ @param head_end_offset - (u16) offset relative to current position
+ where packet head ends
+ @param offset - (i16) copy packet head at current position if 0,
+ else at offset position to change headroom space as specified
+ @return - (u16) number of buffers actually cloned, may be
+ less than the number requested or zero
+*/
+always_inline u16
+vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+ u16 n_buffers, u16 head_end_offset, i16 offset)
+{
+ vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
+ u16 n_cloned = 0;
+
+ while (n_buffers > 256)
+ {
+ vlib_buffer_t *copy;
+ copy = vlib_buffer_copy (vm, s);
+ n_cloned += vlib_buffer_clone_256 (vm,
+ vlib_get_buffer_index (vm, copy),
+ (buffers + n_cloned),
+ 256, head_end_offset, offset);
+ n_buffers -= 256;
+ }
+ n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
+ buffers + n_cloned,
+ n_buffers, head_end_offset, offset);
+
+ return n_cloned;
+}
+
+/** \brief Create multiple clones of buffer and store them
+ in the supplied array
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param src_buffer - (u32) source buffer index
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u16) number of buffer clones requested (<=256)
+ @param head_end_offset - (u16) offset relative to current position
+ where packet head ends
+ @return - (u16) number of buffers actually cloned, may be
+ less than the number requested or zero
+*/
+always_inline u16
+vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+ u16 n_buffers, u16 head_end_offset)
+{
+ return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
+ head_end_offset, 0);
+}
+
/** \brief Attach cloned tail to the buffer
@param vm - (vlib_main_t *) vlib main data structure pointer
vlib_buffer_t * tail)
{
ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
- ASSERT (head->free_list_index == tail->free_list_index);
+ ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
head->flags |= VLIB_BUFFER_NEXT_PRESENT;
head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
tail->total_length_not_including_first_buffer;
next_segment:
- __sync_add_and_fetch (&tail->n_add_refs, 1);
+ clib_atomic_add_fetch (&tail->ref_count, 1);
if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
{
/* The provided next_bi buffer index is appended to the end of the packet. */
always_inline vlib_buffer_t *
-vlib_buffer_chain_buffer (vlib_main_t * vm,
- vlib_buffer_t * first,
- vlib_buffer_t * last, u32 next_bi)
+vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
{
vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
last->next_buffer = next_bi;
* Returns the number of copied bytes. */
always_inline u16
vlib_buffer_chain_append_data (vlib_main_t * vm,
- u32 free_list_index,
vlib_buffer_t * first,
vlib_buffer_t * last, void *data, u16 data_len)
{
- u32 n_buffer_bytes =
- vlib_buffer_free_list_buffer_size (vm, free_list_index);
+ u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
u16 len = clib_min (data_len,
n_buffer_bytes - last->current_length -
last->current_data);
- clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
- len);
+ clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
+ data, len);
vlib_buffer_chain_increase_length (first, last, len);
return len;
}
* chained and points to the last buffer in the chain. */
u16
vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
- u32 free_list_index,
vlib_buffer_t * first,
- vlib_buffer_t ** last,
- void *data, u16 data_len);
+ vlib_buffer_t ** last, void *data,
+ u16 data_len);
void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);
format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
- format_vlib_buffer_contents;
+ format_vlib_buffer_contents, format_vlib_buffer_no_chain;
typedef struct
{
/* Vector of packet data. */
u8 *packet_data;
- /* Number of buffers to allocate in each call to physmem
- allocator. */
- u32 min_n_buffers_each_physmem_alloc;
+ /* Number of buffers to allocate in each call to allocator. */
+ u32 min_n_buffers_each_alloc;
- /* Buffer free list for this template. */
- u32 free_list_index;
-
- u32 *free_buffers;
+ u8 *name;
} vlib_packet_template_t;
-void vlib_packet_template_get_packet_helper (vlib_main_t * vm,
- vlib_packet_template_t * t);
-
void vlib_packet_template_init (vlib_main_t * vm,
vlib_packet_template_t * t,
void *packet_data,
uword n_packet_data_bytes,
- uword min_n_buffers_each_physmem_alloc,
+ uword min_n_buffers_each_alloc,
char *fmt, ...);
void *vlib_packet_template_get_packet (vlib_main_t * vm,
}
always_inline u32
-unserialize_vlib_buffer_n_bytes (serialize_main_t * m)
+vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b)
{
- serialize_stream_t *s = &m->stream;
- vlib_serialize_buffer_main_t *sm
- = uword_to_pointer (m->stream.data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vlib_main_t *vm = sm->vlib_main;
- u32 n, *f;
-
- n = s->n_buffer_bytes - s->current_buffer_index;
- if (sm->last_buffer != ~0)
- {
- vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer);
- while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
- {
- b = vlib_get_buffer (vm, b->next_buffer);
- n += b->current_length;
- }
- }
-
- /* *INDENT-OFF* */
- clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
- n += vlib_buffer_index_length_in_chain (vm, f[0]);
- }));
-/* *INDENT-ON* */
-
- return n;
+ return b->data + vlib_buffer_get_default_data_size (vm) -
+ ((u8 *) vlib_buffer_get_current (b) + b->current_length);
}
-/* Set a buffer quickly into "uninitialized" state. We want this to
- be extremely cheap and arrange for all fields that need to be
- initialized to be in the first 128 bits of the buffer. */
-always_inline void
-vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
- vlib_buffer_free_list_t * fl)
-{
- vlib_buffer_t *src = &fl->buffer_init_template;
-
- /* Make sure vlib_buffer_t is cacheline aligned and sized */
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
- CLIB_CACHE_LINE_BYTES);
- ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
- CLIB_CACHE_LINE_BYTES * 2);
+#define VLIB_BUFFER_LINEARIZE_MAX 64
- /* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
+always_inline u32
+vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
+{
+ vlib_buffer_t *dst_b;
+ u32 n_buffers = 1, to_free = 0;
+ u16 rem_len, dst_len, data_size, src_len = 0;
+ u8 *dst, *src = 0;
- clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ return 1;
- /* Not in the first 16 octets. */
- dst->n_add_refs = src->n_add_refs;
+ ASSERT (1 == b->ref_count);
+ if (PREDICT_FALSE (1 != b->ref_count))
+ return 0;
- /* Make sure it really worked. */
-#define _(f) ASSERT (dst->f == src->f);
- _(current_data);
- _(current_length);
- _(flags);
- _(free_list_index);
-#undef _
- ASSERT (dst->total_length_not_including_first_buffer == 0);
- ASSERT (dst->n_add_refs == 0);
-}
+ data_size = vlib_buffer_get_default_data_size (vm);
+ rem_len = vlib_buffer_length_in_chain (vm, b) - b->current_length;
-always_inline void
-vlib_buffer_add_to_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * f,
- u32 buffer_index, u8 do_init)
-{
- vlib_buffer_t *b;
- b = vlib_get_buffer (vm, buffer_index);
- if (PREDICT_TRUE (do_init))
- vlib_buffer_init_for_free_list (b, f);
- vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
-}
-
-always_inline void
-vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0,
- vlib_buffer_t * dst1,
- vlib_buffer_free_list_t * fl)
-{
- vlib_buffer_t *src = &fl->buffer_init_template;
-
- /* Make sure buffer template is sane. */
- ASSERT (fl->index == fl->buffer_init_template.free_list_index);
-
- clib_memcpy (STRUCT_MARK_PTR (dst0, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
-
- clib_memcpy (STRUCT_MARK_PTR (dst1, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
-
- /* Not in the first 16 octets. */
- dst0->n_add_refs = src->n_add_refs;
- dst1->n_add_refs = src->n_add_refs;
-
- /* Make sure it really worked. */
-#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f)
- _(current_data);
- _(current_length);
- _(flags);
- _(free_list_index);
-#undef _
-
- ASSERT (dst0->total_length_not_including_first_buffer == 0);
- ASSERT (dst1->total_length_not_including_first_buffer == 0);
- ASSERT (dst0->n_add_refs == 0);
- ASSERT (dst1->n_add_refs == 0);
-}
-
-#if CLIB_DEBUG > 0
-extern u32 *vlib_buffer_state_validation_lock;
-extern uword *vlib_buffer_state_validation_hash;
-extern void *vlib_buffer_state_heap;
-#endif
+ dst_b = b;
+ dst = vlib_buffer_get_tail (dst_b);
+ dst_len = vlib_buffer_space_left_at_end (vm, dst_b);
-static inline void
-vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected)
-{
-#if CLIB_DEBUG > 0
- uword *p;
- void *oldheap;
+ b->total_length_not_including_first_buffer -= dst_len;
- oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
+ while (rem_len > 0)
+ {
+ u16 copy_len;
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
- ;
+ while (0 == src_len)
+ {
+ ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
+ if (PREDICT_FALSE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
+ break; /* malformed chained buffer */
- p = hash_get (vlib_buffer_state_validation_hash, b);
+ b = vlib_get_buffer (vm, b->next_buffer);
+ src = vlib_buffer_get_current (b);
+ src_len = b->current_length;
+ }
- /* If we don't know about b, declare it to be in the expected state */
- if (!p)
- {
- hash_set (vlib_buffer_state_validation_hash, b, expected);
- goto out;
- }
+ if (0 == dst_len)
+ {
+ ASSERT (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT);
+ if (PREDICT_FALSE (!(dst_b->flags & VLIB_BUFFER_NEXT_PRESENT)))
+ break; /* malformed chained buffer */
+
+ vlib_buffer_t *next_dst_b = vlib_get_buffer (vm, dst_b->next_buffer);
+
+ if (PREDICT_TRUE (1 == next_dst_b->ref_count))
+ {
+ /* normal case: buffer is not cloned, just use it */
+ dst_b = next_dst_b;
+ }
+ else
+ {
+ /* cloned buffer, build a new dest chain from there */
+ vlib_buffer_t *bufs[VLIB_BUFFER_LINEARIZE_MAX];
+ u32 bis[VLIB_BUFFER_LINEARIZE_MAX + 1];
+ const int n = (rem_len + data_size - 1) / data_size;
+ int n_alloc;
+ int i;
+
+ ASSERT (n <= VLIB_BUFFER_LINEARIZE_MAX);
+ if (PREDICT_FALSE (n > VLIB_BUFFER_LINEARIZE_MAX))
+ return 0;
+
+ n_alloc = vlib_buffer_alloc (vm, bis, n);
+ if (PREDICT_FALSE (n_alloc != n))
+ {
+ vlib_buffer_free (vm, bis, n_alloc);
+ return 0;
+ }
+
+ vlib_get_buffers (vm, bis, bufs, n);
+
+ for (i = 0; i < n - 1; i++)
+ {
+ bufs[i]->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ bufs[i]->next_buffer = bis[i + 1];
+ }
+
+ to_free = dst_b->next_buffer;
+ dst_b->next_buffer = bis[0];
+ dst_b = bufs[0];
+ }
+
+ n_buffers++;
+
+ dst_b->current_data = clib_min (0, dst_b->current_data);
+ dst_b->current_length = 0;
+
+ dst = dst_b->data + dst_b->current_data;
+ dst_len = data_size - dst_b->current_data;
+ }
- if (p[0] != expected)
- {
- void cj_stop (void);
- u32 bi;
- vlib_main_t *vm = &vlib_global_main;
+ copy_len = clib_min (src_len, dst_len);
- cj_stop ();
+ if (PREDICT_TRUE (src == dst))
+ {
+ /* nothing to do */
+ }
+ else if (src + copy_len > dst && dst + copy_len > src)
+ {
+ /* src and dst overlap */
+ ASSERT (b == dst_b);
+ memmove (dst, src, copy_len);
+ }
+ else
+ {
+ clib_memcpy_fast (dst, src, copy_len);
+ }
- bi = vlib_get_buffer_index (vm, b);
+ dst_b->current_length += copy_len;
- clib_mem_set_heap (oldheap);
- clib_warning ("%.6f buffer %llx (%d): %s, not %s",
- vlib_time_now (vm), bi,
- p[0] ? "busy" : "free", expected ? "busy" : "free");
- os_panic ();
+ dst += copy_len;
+ src += copy_len;
+ dst_len -= copy_len;
+ src_len -= copy_len;
+ rem_len -= copy_len;
}
-out:
- CLIB_MEMORY_BARRIER ();
- *vlib_buffer_state_validation_lock = 0;
- clib_mem_set_heap (oldheap);
-#endif
-}
-
-static inline void
-vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected)
-{
-#if CLIB_DEBUG > 0
- void *oldheap;
- oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
+ /* in case of a malformed chain buffer, we'll exit early from the loop. */
+ ASSERT (0 == rem_len);
+ b->total_length_not_including_first_buffer -= rem_len;
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
- ;
+ if (to_free)
+ vlib_buffer_free_one (vm, to_free);
- hash_set (vlib_buffer_state_validation_hash, b, expected);
+ if (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ /* the resulting chain is smaller than the original, cut it there */
+ dst_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ vlib_buffer_free_one (vm, dst_b->next_buffer);
+ if (1 == n_buffers)
+ {
+ /* no longer a chained buffer */
+ dst_b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ dst_b->total_length_not_including_first_buffer = 0;
+ }
+ }
- CLIB_MEMORY_BARRIER ();
- *vlib_buffer_state_validation_lock = 0;
- clib_mem_set_heap (oldheap);
-#endif
+ return n_buffers;
}
#endif /* included_vlib_buffer_funcs_h */