X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer_funcs.h;h=77964fde82121676325fb6c52384f14d0f794b38;hb=e09a2337b82d2dcb5b7379a9581477af291d1a75;hp=08f5b3e901044ab92e8a6ea4f945253680480904;hpb=c8a26c6d239554bac96c481c840b3f5b3d8a17af;p=vpp.git diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 08f5b3e9010..77964fde821 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -41,11 +41,65 @@ #define included_vlib_buffer_funcs_h #include +#include +#include +#include +#include +#include /** \file vlib buffer access methods. */ +typedef void (vlib_buffer_enqueue_to_next_fn_t) (vlib_main_t *vm, + vlib_node_runtime_t *node, + u32 *buffers, u16 *nexts, + uword count); +typedef void (vlib_buffer_enqueue_to_single_next_fn_t) ( + vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index, + u32 count); + +typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) ( + vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, + u32 *buffer_indices, u16 *thread_indices, u32 n_packets, + int drop_on_congestion); + +typedef u32 (vlib_frame_queue_dequeue_fn_t) (vlib_main_t *vm, + vlib_frame_queue_main_t *fqm); + +typedef struct +{ + vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn; + vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn; + vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn; + vlib_frame_queue_dequeue_fn_t *frame_queue_dequeue_fn; +} vlib_buffer_func_main_t; + +extern vlib_buffer_func_main_t vlib_buffer_func_main; + +always_inline void +vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; + + /* reference count in allocated buffer always must be 1 or higher */ + ASSERT (b->ref_count > 0); + + /* verify that buffer pool index is valid */ + bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index); + ASSERT (pointer_to_uword (b) >= bp->start); + ASSERT (pointer_to_uword (b) < bp->start + bp->size - + (bp->data_size + sizeof (vlib_buffer_t))); +} + +always_inline void * +vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index, + uword offset) +{ + offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; + return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *); +} /** \brief Translate buffer index into buffer pointer @@ -57,10 +111,206 @@ always_inline vlib_buffer_t * vlib_get_buffer (vlib_main_t * vm, u32 buffer_index) { vlib_buffer_main_t *bm = vm->buffer_main; - uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; - ASSERT (offset < bm->buffer_mem_size); + vlib_buffer_t *b; + + b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0); + vlib_buffer_validate (vm, b); + return b; +} + +static_always_inline u32 +vlib_buffer_get_default_data_size (vlib_main_t * vm) +{ + return vm->buffer_main->default_data_size; +} + +static_always_inline void +vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices) +{ + clib_memcpy_u32 (dst, src, n_indices); +} + +always_inline void +vlib_buffer_copy_indices_from_ring (u32 * dst, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers) +{ + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + { + vlib_buffer_copy_indices (dst, ring + start, n_buffers); + } + else + { + u32 n = ring_size - start; + vlib_buffer_copy_indices (dst, ring + start, n); + vlib_buffer_copy_indices (dst + n, ring, n_buffers - n); + } +} + +always_inline void +vlib_buffer_copy_indices_to_ring (u32 * ring, u32 * src, u32 start, + u32 ring_size, u32 n_buffers) +{ + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + { + vlib_buffer_copy_indices (ring + start, src, n_buffers); + } + else + { + u32 n = ring_size - start; + vlib_buffer_copy_indices (ring + start, src, n); + vlib_buffer_copy_indices (ring, src + n, n_buffers - n); + } +} - return uword_to_pointer (bm->buffer_mem_start + offset, void *); +STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); +static_always_inline void +vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt) +{ +#if defined CLIB_HAVE_VEC512 + b->as_u8x64[0] = bt->as_u8x64[0]; +#elif defined (CLIB_HAVE_VEC256) + b->as_u8x32[0] = bt->as_u8x32[0]; + b->as_u8x32[1] = bt->as_u8x32[1]; +#elif defined (CLIB_HAVE_VEC128) + b->as_u8x16[0] = bt->as_u8x16[0]; + b->as_u8x16[1] = bt->as_u8x16[1]; + b->as_u8x16[2] = bt->as_u8x16[2]; + b->as_u8x16[3] = bt->as_u8x16[3]; +#else + clib_memcpy_fast (b, bt, 64); +#endif +} + +always_inline u8 +vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node) +{ + ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES); + return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node]; +} + +/** \brief Translate array of buffer indices into buffer pointers with offset + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param bi - (u32 *) array of buffer indices + @param b - (void **) array to store buffer pointers + @param count - (uword) number of elements + @param offset - (i32) offset applied to each pointer +*/ +static_always_inline void +vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, + i32 offset) +{ + uword buffer_mem_start = vm->buffer_main->buffer_mem_start; +#ifdef CLIB_HAVE_VEC512 + u64x8 of8 = u64x8_splat (buffer_mem_start + offset); + u64x4 off = u64x8_extract_lo (of8); + /* if count is not const, compiler will not unroll while loop + se we maintain two-in-parallel variant */ + while (count >= 32) + { + u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi)); + u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8)); + u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16)); + u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24)); + /* shift and add to get vlib_buffer_t pointer */ + u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b); + u64x8_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 8); + u64x8_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 16); + u64x8_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 24); + b += 32; + bi += 32; + count -= 32; + } + while (count >= 8) + { + u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi)); + /* shift and add to get vlib_buffer_t pointer */ + u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b); + b += 8; + bi += 8; + count -= 8; + } +#elif defined CLIB_HAVE_VEC256 + u64x4 off = u64x4_splat (buffer_mem_start + offset); + /* if count is not const, compiler will not unroll while loop + se we maintain two-in-parallel variant */ + while (count >= 32) + { + u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi)); + u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4)); + u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8)); + u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12)); + u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16)); + u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20)); + u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24)); + u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28)); + /* shift and add to get vlib_buffer_t pointer */ + u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); + u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4); + u64x4_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 8); + u64x4_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 12); + u64x4_store_unaligned ((b4 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 16); + u64x4_store_unaligned ((b5 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 20); + u64x4_store_unaligned ((b6 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 24); + u64x4_store_unaligned ((b7 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 28); + b += 32; + bi += 32; + count -= 32; + } +#endif + while (count >= 4) + { +#ifdef CLIB_HAVE_VEC256 + u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi)); + /* shift and add to get vlib_buffer_t pointer */ + u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); +#elif defined (CLIB_HAVE_VEC128) + u64x2 off = u64x2_splat (buffer_mem_start + offset); + u32x4 bi4 = u32x4_load_unaligned (bi); + u64x2 b0 = u64x2_from_u32x4 ((u32x4) bi4); +#if defined (__aarch64__) + u64x2 b1 = u64x2_from_u32x4_high ((u32x4) bi4); +#else + bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1); + u64x2 b1 = u64x2_from_u32x4 ((u32x4) bi4); +#endif + u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); + u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2); +#else + b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset); + b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset); + b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset); + b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset); +#endif + b += 4; + bi += 4; + count -= 4; + } + while (count) + { + b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset); + b += 1; + bi += 1; + count -= 1; + } +} + +/** \brief Translate array of buffer indices into buffer pointers + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param bi - (u32 *) array of buffer indices + @param b - (vlib_buffer_t **) array to store buffer pointers + @param count - (uword) number of elements +*/ + +static_always_inline void +vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count) +{ + vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0); } /** \brief Translate buffer pointer into buffer index @@ -81,6 +331,82 @@ vlib_get_buffer_index (vlib_main_t * vm, void *p) return offset >> CLIB_LOG2_CACHE_LINE_BYTES; } +/** \brief Translate array of buffer pointers into buffer indices with offset + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param b - (void **) array of buffer pointers + @param bi - (u32 *) array to store buffer indices + @param count - (uword) number of elements + @param offset - (i32) offset applied to each pointer +*/ +static_always_inline void +vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi, + uword count, i32 offset) +{ +#ifdef CLIB_HAVE_VEC256 + u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 }; + u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset); + + while (count >= 8) + { + /* load 4 pointers into 256-bit register */ + u64x4 v0 = u64x4_load_unaligned (b); + u64x4 v1 = u64x4_load_unaligned (b + 4); + u32x8 v2, v3; + + v0 -= off4; + v1 -= off4; + + v0 >>= CLIB_LOG2_CACHE_LINE_BYTES; + v1 >>= CLIB_LOG2_CACHE_LINE_BYTES; + + /* permute 256-bit register so lower u32s of each buffer index are + * placed into lower 128-bits */ + v2 = u32x8_permute ((u32x8) v0, mask); + v3 = u32x8_permute ((u32x8) v1, mask); + + /* extract lower 128-bits and save them to the array of buffer indices */ + u32x4_store_unaligned (u32x8_extract_lo (v2), bi); + u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4); + bi += 8; + b += 8; + count -= 8; + } +#endif + while (count >= 4) + { + /* equivalent non-nector implementation */ + bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset); + bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset); + bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset); + bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset); + bi += 4; + b += 4; + count -= 4; + } + while (count) + { + bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset); + bi += 1; + b += 1; + count -= 1; + } +} + +/** \brief Translate array of buffer pointers into buffer indices + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param b - (vlib_buffer_t **) array of buffer pointers + @param bi - (u32 *) array to store buffer indices + @param count - (uword) number of elements +*/ +static_always_inline void +vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi, + uword count) +{ + vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0); +} + /** \brief Get next buffer in buffer linklist, or zero for end of list. @param vm - (vlib_main_t *) vlib main data structure pointer @@ -148,7 +474,7 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) { b = vlib_get_buffer (vm, buffer_index); l = b->current_length; - clib_memcpy (contents + content_len, b->data + b->current_data, l); + clib_memcpy_fast (contents + content_len, b->data + b->current_data, l); content_len += l; if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) break; @@ -158,18 +484,16 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) return content_len; } -/* Return physical address of buffer->data start. */ -always_inline u64 -vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) +always_inline uword +vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b) +{ + return vlib_physmem_get_pa (vm, b->data); +} + +always_inline uword +vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b) { - vlib_physmem_region_index_t pri; - vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index); - pri = vm->buffer_main->buffer_pools[b->buffer_pool_index].physmem_region; - return vlib_physmem_offset_to_physical (vm, pri, - (((uword) buffer_index) << - CLIB_LOG2_CACHE_LINE_BYTES) + - STRUCT_OFFSET_OF (vlib_buffer_t, - data)); + return vlib_buffer_get_pa (vm, b) + b->current_data; } /** \brief Prefetch buffer metadata by buffer index @@ -186,28 +510,6 @@ vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) vlib_prefetch_buffer_header (_b, type); \ } while (0) -#if 0 -/* Iterate over known allocated vlib bufs. You probably do not want - * to do this! - @param vm the vlib_main_t - @param bi found allocated buffer index - @param body operation to perform on buffer index - function executes body for each allocated buffer index - */ -#define vlib_buffer_foreach_allocated(vm,bi,body) \ -do { \ - vlib_main_t * _vmain = (vm); \ - vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \ - hash_pair_t * _vbpair; \ - hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \ - if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \ - (bi) = _vbpair->key; \ - body; \ - } \ - })); \ -} while (0) -#endif - typedef enum { /* Index is unknown. */ @@ -234,47 +536,55 @@ vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) return p ? p[0] : VLIB_BUFFER_UNKNOWN; } -always_inline void -vlib_buffer_set_known_state (vlib_main_t * vm, - u32 buffer_index, - vlib_buffer_known_state_t state) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - clib_spinlock_lock (&bm->buffer_known_hash_lockp); - hash_set (bm->buffer_known_hash, buffer_index, state); - clib_spinlock_unlock (&bm->buffer_known_hash_lockp); -} - /* Validates sanity of a single buffer. Returns format'ed vector with error message if any. */ u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -always_inline u32 -vlib_buffer_round_size (u32 size) -{ - return round_pow2 (size, sizeof (vlib_buffer_t)); -} +u8 *vlib_validate_buffers (vlib_main_t * vm, + u32 * buffers, + uword next_buffer_stride, + uword n_buffers, + vlib_buffer_known_state_t known_state, + uword follow_buffer_next); -always_inline u32 -vlib_buffer_get_free_list_index (vlib_buffer_t * b) +static_always_inline vlib_buffer_pool_t * +vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) { - return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK; + vlib_buffer_main_t *bm = vm->buffer_main; + return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); } -always_inline void -vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) +static_always_inline __clib_warn_unused_result uword +vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, + u32 n_buffers) { - /* if there is an need for more free lists we should consider - storig data in the 2nd cacheline */ - ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1); - ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK); + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + u32 len; + + ASSERT (bp->buffers); - b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK; - b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK; + clib_spinlock_lock (&bp->lock); + len = bp->n_avail; + if (PREDICT_TRUE (n_buffers < len)) + { + len -= n_buffers; + vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers); + bp->n_avail = len; + clib_spinlock_unlock (&bp->lock); + return n_buffers; + } + else + { + vlib_buffer_copy_indices (buffers, bp->buffers, len); + bp->n_avail = 0; + clib_spinlock_unlock (&bp->lock); + return len; + } } -/** \brief Allocate buffers from specific freelist into supplied array + +/** \brief Allocate buffers from specific pool into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @@ -282,55 +592,103 @@ vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -always_inline u32 -vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) + +always_inline __clib_warn_unused_result u32 +vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u8 buffer_pool_index) { vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *fl; - u32 *src; - uword len; + vlib_buffer_pool_t *bp; + vlib_buffer_pool_thread_t *bpt; + u32 *src, *dst, len, n_left; + + /* If buffer allocation fault injection is configured */ + if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0) + { + u32 vlib_buffer_alloc_may_fail (vlib_main_t *, u32); - ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + /* See how many buffers we're willing to allocate */ + n_buffers = vlib_buffer_alloc_may_fail (vm, n_buffers); + if (n_buffers == 0) + return (n_buffers); + } - fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index); + bpt = vec_elt_at_index (bp->threads, vm->thread_index); - len = vec_len (fl->buffers); + dst = buffers; + n_left = n_buffers; + len = bpt->n_cached; - if (PREDICT_FALSE (len < n_buffers)) + /* per-thread cache contains enough buffers */ + if (len >= n_buffers) { - bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); - len = vec_len (fl->buffers); + src = bpt->cached_buffers + len - n_buffers; + vlib_buffer_copy_indices (dst, src, n_buffers); + bpt->n_cached -= n_buffers; + goto done; + } - /* even if fill free list didn't manage to refill free list - we should give what we have */ - n_buffers = clib_min (len, n_buffers); + /* alloc bigger than cache - take buffers directly from main pool */ + if (n_buffers >= VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ) + { + n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers, + n_buffers); + goto done; + } - /* following code is intentionaly duplicated to allow compiler - to optimize fast path when n_buffers is constant value */ - src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); - _vec_len (fl->buffers) -= n_buffers; + /* take everything available in the cache */ + if (len) + { + vlib_buffer_copy_indices (dst, bpt->cached_buffers, len); + bpt->n_cached = 0; + dst += len; + n_left -= len; + } - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); + len = round_pow2 (n_left, 32); + len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers, + len); + bpt->n_cached = len; - return n_buffers; + if (len) + { + u32 n_copy = clib_min (len, n_left); + src = bpt->cached_buffers + len - n_copy; + vlib_buffer_copy_indices (dst, src, n_copy); + bpt->n_cached -= n_copy; + n_left -= n_copy; } - src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); - _vec_len (fl->buffers) -= n_buffers; + n_buffers -= n_left; +done: /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); - + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + if (PREDICT_FALSE (bm->alloc_callback_fn != 0)) + bm->alloc_callback_fn (vm, buffer_pool_index, buffers, n_buffers); return n_buffers; } +/** \brief Allocate buffers from specific numa node into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @param numa_node - (u32) numa node + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline __clib_warn_unused_result u32 +vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u32 numa_node) +{ + u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node); + return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index); +} + /** \brief Allocate buffers into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @@ -339,13 +697,276 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -always_inline u32 + +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { - return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node); +} + +/** \brief Allocate buffers into ring + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline __clib_warn_unused_result u32 +vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers) +{ + u32 n_alloc; + + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + return vlib_buffer_alloc (vm, ring + start, n_buffers); + + n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start); + + if (PREDICT_TRUE (n_alloc == ring_size - start)) + n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc); + + return n_alloc; } +/** \brief Allocate buffers into ring from specific buffer pool + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline __clib_warn_unused_result u32 +vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers, + u8 buffer_pool_index) +{ + u32 n_alloc; + + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers, + buffer_pool_index); + + n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start, + buffer_pool_index); + + if (PREDICT_TRUE (n_alloc == ring_size - start)) + n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc, + buffer_pool_index); + + return n_alloc; +} + +static_always_inline void +vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index, + u32 * buffers, u32 n_buffers) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads, + vm->thread_index); + u32 n_cached, n_empty; + + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_ALLOCATED); + if (PREDICT_FALSE (bm->free_callback_fn != 0)) + bm->free_callback_fn (vm, buffer_pool_index, buffers, n_buffers); + + n_cached = bpt->n_cached; + n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached; + if (n_buffers <= n_empty) + { + vlib_buffer_copy_indices (bpt->cached_buffers + n_cached, + buffers, n_buffers); + bpt->n_cached = n_cached + n_buffers; + return; + } + + vlib_buffer_copy_indices (bpt->cached_buffers + n_cached, + buffers + n_buffers - n_empty, n_empty); + bpt->n_cached = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ; + + clib_spinlock_lock (&bp->lock); + vlib_buffer_copy_indices (bp->buffers + bp->n_avail, buffers, + n_buffers - n_empty); + bp->n_avail += n_buffers - n_empty; + clib_spinlock_unlock (&bp->lock); +} + +static_always_inline void +vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + int maybe_next) +{ + const int queue_size = 128; + vlib_buffer_pool_t *bp = 0; + u8 buffer_pool_index = ~0; + u32 n_queue = 0, queue[queue_size + 4]; + vlib_buffer_t bt = { }; +#if defined(CLIB_HAVE_VEC128) + vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; + vlib_buffer_t bpi_vec = {}; + vlib_buffer_t flags_refs_mask = { + .flags = VLIB_BUFFER_NEXT_PRESENT, + .ref_count = ~1 + }; +#endif + + if (PREDICT_FALSE (n_buffers == 0)) + return; + + vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]); + buffer_pool_index = b->buffer_pool_index; + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); +#if defined(CLIB_HAVE_VEC128) + bpi_vec.buffer_pool_index = buffer_pool_index; +#endif + + while (n_buffers) + { + vlib_buffer_t *b[8]; + u32 bi, sum = 0, flags, next; + + if (n_buffers < 4) + goto one_by_one; + + vlib_get_buffers (vm, buffers, b, 4); + + if (n_buffers >= 12) + { + vlib_get_buffers (vm, buffers + 8, b + 4, 4); + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + } + +#if defined(CLIB_HAVE_VEC128) + u8x16 p0, p1, p2, p3, r; + p0 = u8x16_load_unaligned (b[0]); + p1 = u8x16_load_unaligned (b[1]); + p2 = u8x16_load_unaligned (b[2]); + p3 = u8x16_load_unaligned (b[3]); + + r = p0 ^ bpi_vec.as_u8x16[0]; + r |= p1 ^ bpi_vec.as_u8x16[0]; + r |= p2 ^ bpi_vec.as_u8x16[0]; + r |= p3 ^ bpi_vec.as_u8x16[0]; + r &= bpi_mask.as_u8x16[0]; + r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0]; + + sum = !u8x16_is_all_zero (r); +#else + sum |= b[0]->flags; + sum |= b[1]->flags; + sum |= b[2]->flags; + sum |= b[3]->flags; + sum &= VLIB_BUFFER_NEXT_PRESENT; + sum += b[0]->ref_count - 1; + sum += b[1]->ref_count - 1; + sum += b[2]->ref_count - 1; + sum += b[3]->ref_count - 1; + sum |= b[0]->buffer_pool_index ^ buffer_pool_index; + sum |= b[1]->buffer_pool_index ^ buffer_pool_index; + sum |= b[2]->buffer_pool_index ^ buffer_pool_index; + sum |= b[3]->buffer_pool_index ^ buffer_pool_index; +#endif + + if (sum) + goto one_by_one; + + vlib_buffer_copy_indices (queue + n_queue, buffers, 4); + vlib_buffer_copy_template (b[0], &bt); + vlib_buffer_copy_template (b[1], &bt); + vlib_buffer_copy_template (b[2], &bt); + vlib_buffer_copy_template (b[3], &bt); + n_queue += 4; + + vlib_buffer_validate (vm, b[0]); + vlib_buffer_validate (vm, b[1]); + vlib_buffer_validate (vm, b[2]); + vlib_buffer_validate (vm, b[3]); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); + + if (n_queue >= queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + buffers += 4; + n_buffers -= 4; + continue; + + one_by_one: + bi = buffers[0]; + + next_in_chain: + b[0] = vlib_get_buffer (vm, bi); + flags = b[0]->flags; + next = b[0]->next_buffer; + + if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index)) + { + + if (n_queue) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + + buffer_pool_index = b[0]->buffer_pool_index; +#if defined(CLIB_HAVE_VEC128) + bpi_vec.buffer_pool_index = buffer_pool_index; +#endif + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); + } + + vlib_buffer_validate (vm, b[0]); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + + if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0) + { + vlib_buffer_copy_template (b[0], &bt); + queue[n_queue++] = bi; + } + + if (n_queue == queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size); + n_queue = 0; + } + + if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT)) + { + bi = next; + goto next_in_chain; + } + + buffers++; + n_buffers--; + } + + if (n_queue) + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); +} + + /** \brief Free buffers Frees the entire buffer chain for each buffer @@ -361,11 +982,7 @@ vlib_buffer_free (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_cb); - - return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1); } /** \brief Free buffers, does not free the buffer chain for each buffer @@ -382,11 +999,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_no_next_cb); - - return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0); } /** \brief Free one buffer @@ -398,108 +1011,67 @@ vlib_buffer_free_no_next (vlib_main_t * vm, always_inline void vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1); + vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1); } -/* Add/delete buffer free lists. */ -u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...); -always_inline void -vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_delete_free_list_cb); - - bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index); -} - -/* Find already existing public free list with given size or create one. */ -u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...); - -/* Merge two free lists */ -void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst, - vlib_buffer_free_list_t * src); - -/* Make sure we have at least given number of unaligned buffers. */ -void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm, - vlib_buffer_free_list_t * - free_list, - uword n_unaligned_buffers); - -always_inline u32 -vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size) -{ - vlib_buffer_main_t *bm = vm->buffer_main; +/** \brief Free buffers from ring - size = vlib_buffer_round_size (size); - uword *p = hash_get (bm->free_list_by_size, size); - return p ? p[0] : ~0; -} - -always_inline vlib_buffer_free_list_t * -vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b, - u32 * index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - u32 i; - - *index = i = vlib_buffer_get_free_list_index (b); - return pool_elt_at_index (bm->buffer_free_list_pool, i); -} - -always_inline vlib_buffer_free_list_t * -vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index) + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers +*/ +always_inline void +vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - - f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + ASSERT (n_buffers <= ring_size); - /* Sanity: indices must match. */ - ASSERT (f->index == free_list_index); - - return f; -} - -always_inline u32 -vlib_buffer_free_list_buffer_size (vlib_main_t * vm, u32 free_list_index) -{ - vlib_buffer_free_list_t *f = - vlib_buffer_get_free_list (vm, free_list_index); - return f->n_data_bytes; + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + { + vlib_buffer_free (vm, ring + start, n_buffers); + } + else + { + vlib_buffer_free (vm, ring + start, ring_size - start); + vlib_buffer_free (vm, ring, n_buffers - (ring_size - start)); + } } -void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes); +/** \brief Free buffers from ring without freeing tail buffers -/* Reasonably fast buffer copy routine. */ + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers +*/ always_inline void -vlib_copy_buffers (u32 * dst, u32 * src, u32 n) +vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers) { - while (n >= 4) + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - dst[3] = src[3]; - dst += 4; - src += 4; - n -= 4; + vlib_buffer_free_no_next (vm, ring + start, n_buffers); } - while (n > 0) + else { - dst[0] = src[0]; - dst += 1; - src += 1; - n -= 1; + vlib_buffer_free_no_next (vm, ring + start, ring_size - start); + vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start)); } } /* Append given data to end of buffer, possibly allocating new buffers. */ -u32 vlib_buffer_add_data (vlib_main_t * vm, - u32 free_list_index, - u32 buffer_index, void *data, u32 n_data_bytes); +int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, + u32 n_data_bytes); + +/* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */ +#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \ + (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \ + VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL) /* duplicate all buffers in chain */ always_inline vlib_buffer_t * @@ -507,7 +1079,7 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) { vlib_buffer_t *s, *d, *fd; uword n_alloc, n_buffers = 1; - u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID; + u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK; int i; s = b; @@ -534,11 +1106,13 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d->current_data = s->current_data; d->current_length = s->current_length; d->flags = s->flags & flag_mask; + d->trace_handle = s->trace_handle; d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); /* next segments */ for (i = 1; i < n_buffers; i++) @@ -550,39 +1124,94 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d = vlib_get_buffer (vm, new_buffers[i]); d->current_data = s->current_data; d->current_length = s->current_length; - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); d->flags = s->flags & flag_mask; } return fd; } -/** \brief Create multiple clones of buffer and store them in the supplied array +/* duplicate first buffer in chain */ +always_inline vlib_buffer_t * +vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di) +{ + vlib_buffer_t *d; + + if ((vlib_buffer_alloc (vm, di, 1)) != 1) + return 0; + + d = vlib_get_buffer (vm, *di); + /* 1st segment */ + d->current_data = b->current_data; + d->current_length = b->current_length; + clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque)); + clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (b), b->current_length); + + return d; +} + +/* \brief Move packet from current position to offset position in buffer. + Only work for small packet using one buffer with room to fit the move + @param vm - (vlib_main_t *) vlib main data structure pointer + @param b - (vlib_buffer_t *) pointer to buffer + @param offset - (i16) position to move the packet in buffer + */ +always_inline void +vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset) +{ + ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); + ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); + ASSERT (offset + b->current_length < + vlib_buffer_get_default_data_size (vm)); + + u8 *source = vlib_buffer_get_current (b); + b->current_data = offset; + u8 *destination = vlib_buffer_get_current (b); + u16 length = b->current_length; + + if (source + length <= destination) /* no overlap */ + clib_memcpy_fast (destination, source, length); + else + memmove (destination, source, length); +} + +/** \brief Create a maximum of 256 clones of buffer and store them + in the supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param src_buffer - (u32) source buffer index @param buffers - (u32 * ) buffer index array - @param n_buffers - (u8) number of buffer clones requested + @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends - @return - (u8) number of buffers actually cloned, may be + @param offset - (i16) copy packet head at current position if 0, + else at offset position to change headroom space as specified + @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ - -always_inline u8 -vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, - u8 n_buffers, u16 head_end_offset) +always_inline u16 +vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, + u16 n_buffers, u16 head_end_offset, i16 offset) { - u8 i; + u16 i; vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); - ASSERT (s->n_add_refs == 0); + ASSERT (s->ref_count == 1); ASSERT (n_buffers); + ASSERT (n_buffers <= 256); + ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); + ASSERT ((offset + head_end_offset) < + vlib_buffer_get_default_data_size (vm)); if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2) { buffers[0] = src_buffer; + if (offset) + vlib_buffer_move (vm, s, offset); + for (i = 1; i < n_buffers; i++) { vlib_buffer_t *d; @@ -595,43 +1224,111 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, return n_buffers; } - n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - vlib_buffer_get_free_list_index - (s)); - if (PREDICT_FALSE (n_buffers == 0)) + if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0))) { buffers[0] = src_buffer; return 1; } + n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, + s->buffer_pool_index); + for (i = 0; i < n_buffers; i++) { vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); - d->current_data = s->current_data; + if (offset) + d->current_data = offset; + else + d->current_data = s->current_data; + d->current_length = head_end_offset; - vlib_buffer_set_free_list_index (d, - vlib_buffer_get_free_list_index (s)); - d->total_length_not_including_first_buffer = - s->total_length_not_including_first_buffer + s->current_length - + ASSERT (d->buffer_pool_index == s->buffer_pool_index); + + d->total_length_not_including_first_buffer = s->current_length - head_end_offset; - d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT; - d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s), - head_end_offset); + if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT)) + { + d->total_length_not_including_first_buffer += + s->total_length_not_including_first_buffer; + } + d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) | + VLIB_BUFFER_NEXT_PRESENT; + d->trace_handle = s->trace_handle; + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), head_end_offset); d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers ? n_buffers : s->ref_count; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { s = vlib_get_buffer (vm, s->next_buffer); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers ? n_buffers : s->ref_count; } return n_buffers; } +/** \brief Create multiple clones of buffer and store them + in the supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param src_buffer - (u32) source buffer index + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u16) number of buffer clones requested (<=256) + @param head_end_offset - (u16) offset relative to current position + where packet head ends + @param offset - (i16) copy packet head at current position if 0, + else at offset position to change headroom space as specified + @return - (u16) number of buffers actually cloned, may be + less than the number requested or zero +*/ +always_inline u16 +vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers, + u16 n_buffers, u16 head_end_offset, i16 offset) +{ + vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); + u16 n_cloned = 0; + + while (n_buffers > 256) + { + vlib_buffer_t *copy; + copy = vlib_buffer_copy (vm, s); + n_cloned += vlib_buffer_clone_256 (vm, + vlib_get_buffer_index (vm, copy), + (buffers + n_cloned), + 256, head_end_offset, offset); + n_buffers -= 256; + } + n_cloned += vlib_buffer_clone_256 (vm, src_buffer, + buffers + n_cloned, + n_buffers, head_end_offset, offset); + + return n_cloned; +} + +/** \brief Create multiple clones of buffer and store them + in the supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param src_buffer - (u32) source buffer index + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u16) number of buffer clones requested (<=256) + @param head_end_offset - (u16) offset relative to current position + where packet head ends + @return - (u16) number of buffers actually cloned, may be + less than the number requested or zero +*/ +always_inline u16 +vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, + u16 n_buffers, u16 head_end_offset) +{ + return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers, + head_end_offset, 0); +} + /** \brief Attach cloned tail to the buffer @param vm - (vlib_main_t *) vlib main data structure pointer @@ -644,8 +1341,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT (vlib_buffer_get_free_list_index (head) == - vlib_buffer_get_free_list_index (tail)); + ASSERT (head->buffer_pool_index == tail->buffer_pool_index); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -656,7 +1352,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, tail->total_length_not_including_first_buffer; next_segment: - __sync_add_and_fetch (&tail->n_add_refs, 1); + clib_atomic_add_fetch (&tail->ref_count, 1); if (tail->flags & VLIB_BUFFER_NEXT_PRESENT) { @@ -677,9 +1373,7 @@ vlib_buffer_chain_init (vlib_buffer_t * first) /* The provided next_bi buffer index is appended to the end of the packet. */ always_inline vlib_buffer_t * -vlib_buffer_chain_buffer (vlib_main_t * vm, - vlib_buffer_t * first, - vlib_buffer_t * last, u32 next_bi) +vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi) { vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi); last->next_buffer = next_bi; @@ -707,18 +1401,16 @@ vlib_buffer_chain_increase_length (vlib_buffer_t * first, * Returns the number of copied bytes. */ always_inline u16 vlib_buffer_chain_append_data (vlib_main_t * vm, - u32 free_list_index, vlib_buffer_t * first, vlib_buffer_t * last, void *data, u16 data_len) { - u32 n_buffer_bytes = - vlib_buffer_free_list_buffer_size (vm, free_list_index); + u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); ASSERT (n_buffer_bytes >= last->current_length + last->current_data); u16 len = clib_min (data_len, n_buffer_bytes - last->current_length - last->current_data); - clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data, - len); + clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length, + data, len); vlib_buffer_chain_increase_length (first, last, len); return len; } @@ -730,38 +1422,30 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, * chained and points to the last buffer in the chain. */ u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - u32 free_list_index, vlib_buffer_t * first, - vlib_buffer_t ** last, - void *data, u16 data_len); + vlib_buffer_t ** last, void *data, + u16 data_len); void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first); format_function_t format_vlib_buffer, format_vlib_buffer_and_data, - format_vlib_buffer_contents; + format_vlib_buffer_contents, format_vlib_buffer_no_chain; typedef struct { /* Vector of packet data. */ u8 *packet_data; - /* Number of buffers to allocate in each call to physmem - allocator. */ - u32 min_n_buffers_each_physmem_alloc; + /* Number of buffers to allocate in each call to allocator. */ + u32 min_n_buffers_each_alloc; - /* Buffer free list for this template. */ - u32 free_list_index; - - u32 *free_buffers; + u8 *name; } vlib_packet_template_t; -void vlib_packet_template_get_packet_helper (vlib_main_t * vm, - vlib_packet_template_t * t); - void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, void *packet_data, uword n_packet_data_bytes, - uword min_n_buffers_each_physmem_alloc, + uword min_n_buffers_each_alloc, char *fmt, ...); void *vlib_packet_template_get_packet (vlib_main_t * vm, @@ -775,206 +1459,155 @@ vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t) } always_inline u32 -unserialize_vlib_buffer_n_bytes (serialize_main_t * m) +vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b) { - serialize_stream_t *s = &m->stream; - vlib_serialize_buffer_main_t *sm - = uword_to_pointer (m->stream.data_function_opaque, - vlib_serialize_buffer_main_t *); - vlib_main_t *vm = sm->vlib_main; - u32 n, *f; - - n = s->n_buffer_bytes - s->current_buffer_index; - if (sm->last_buffer != ~0) - { - vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer); - while (b->flags & VLIB_BUFFER_NEXT_PRESENT) - { - b = vlib_get_buffer (vm, b->next_buffer); - n += b->current_length; - } - } - - /* *INDENT-OFF* */ - clib_fifo_foreach (f, sm->rx.buffer_fifo, ({ - n += vlib_buffer_index_length_in_chain (vm, f[0]); - })); -/* *INDENT-ON* */ - - return n; + return b->data + vlib_buffer_get_default_data_size (vm) - + ((u8 *) vlib_buffer_get_current (b) + b->current_length); } -/* Set a buffer quickly into "uninitialized" state. We want this to - be extremely cheap and arrange for all fields that need to be - initialized to be in the first 128 bits of the buffer. */ -always_inline void -vlib_buffer_init_for_free_list (vlib_buffer_t * dst, - vlib_buffer_free_list_t * fl) -{ - vlib_buffer_t *src = &fl->buffer_init_template; - - /* Make sure vlib_buffer_t is cacheline aligned and sized */ - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0); - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) == - CLIB_CACHE_LINE_BYTES); - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) == - CLIB_CACHE_LINE_BYTES * 2); - - /* Make sure buffer template is sane. */ - ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - - clib_memcpy (STRUCT_MARK_PTR (dst, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - /* Not in the first 16 octets. */ - dst->n_add_refs = src->n_add_refs; - - /* Make sure it really worked. */ -#define _(f) ASSERT (dst->f == src->f); - _(current_data); - _(current_length); - _(flags); -#undef _ - /* ASSERT (dst->total_length_not_including_first_buffer == 0); */ - /* total_length_not_including_first_buffer is not in the template anymore - * so it may actually not zeroed for some buffers. One option is to - * uncomment the line lower (comes at a cost), the other, is to just not - * care */ - /* dst->total_length_not_including_first_buffer = 0; */ - ASSERT (dst->n_add_refs == 0); -} +#define VLIB_BUFFER_LINEARIZE_MAX 64 -always_inline void -vlib_buffer_add_to_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * f, - u32 buffer_index, u8 do_init) +always_inline u32 +vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b) { - vlib_buffer_t *b; - b = vlib_get_buffer (vm, buffer_index); - if (PREDICT_TRUE (do_init)) - vlib_buffer_init_for_free_list (b, f); - vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES); - - if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE) - { - vlib_buffer_free_list_t *mf; - mf = vlib_buffer_get_free_list (vlib_mains[0], f->index); - clib_spinlock_lock (&mf->global_buffers_lock); - /* keep last stored buffers, as they are more likely hot in the cache */ - vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE, - CLIB_CACHE_LINE_BYTES); - vec_delete (f->buffers, VLIB_FRAME_SIZE, 0); - f->n_alloc -= VLIB_FRAME_SIZE; - clib_spinlock_unlock (&mf->global_buffers_lock); - } -} + vlib_buffer_t *dst_b; + u32 n_buffers = 1, to_free = 0; + u16 rem_len, dst_len, data_size, src_len = 0; + u8 *dst, *src = 0; -always_inline void -vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, - vlib_buffer_t * dst1, - vlib_buffer_free_list_t * fl) -{ - vlib_buffer_t *src = &fl->buffer_init_template; - - /* Make sure buffer template is sane. */ - ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - - clib_memcpy (STRUCT_MARK_PTR (dst0, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - clib_memcpy (STRUCT_MARK_PTR (dst1, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - /* Not in the first 16 octets. */ - dst0->n_add_refs = src->n_add_refs; - dst1->n_add_refs = src->n_add_refs; - - /* Make sure it really worked. */ -#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f) - _(current_data); - _(current_length); - _(flags); -#undef _ - - ASSERT (dst0->total_length_not_including_first_buffer == 0); - ASSERT (dst1->total_length_not_including_first_buffer == 0); - ASSERT (dst0->n_add_refs == 0); - ASSERT (dst1->n_add_refs == 0); -} - -#if CLIB_DEBUG > 0 -extern u32 *vlib_buffer_state_validation_lock; -extern uword *vlib_buffer_state_validation_hash; -extern void *vlib_buffer_state_heap; -#endif + if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + return 1; -static inline void -vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - uword *p; - void *oldheap; + ASSERT (1 == b->ref_count); + if (PREDICT_FALSE (1 != b->ref_count)) + return 0; - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); + data_size = vlib_buffer_get_default_data_size (vm); + rem_len = vlib_buffer_length_in_chain (vm, b) - b->current_length; - while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1)) - ; + dst_b = b; + dst = vlib_buffer_get_tail (dst_b); + dst_len = vlib_buffer_space_left_at_end (vm, dst_b); - p = hash_get (vlib_buffer_state_validation_hash, b); + b->total_length_not_including_first_buffer -= dst_len; - /* If we don't know about b, declare it to be in the expected state */ - if (!p) + while (rem_len > 0) { - hash_set (vlib_buffer_state_validation_hash, b, expected); - goto out; - } + u16 copy_len; - if (p[0] != expected) - { - void cj_stop (void); - u32 bi; - vlib_main_t *vm = &vlib_global_main; + while (0 == src_len) + { + ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT); + if (PREDICT_FALSE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))) + break; /* malformed chained buffer */ - cj_stop (); + b = vlib_get_buffer (vm, b->next_buffer); + src = vlib_buffer_get_current (b); + src_len = b->current_length; + } - bi = vlib_get_buffer_index (vm, b); + if (0 == dst_len) + { + ASSERT (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT); + if (PREDICT_FALSE (!(dst_b->flags & VLIB_BUFFER_NEXT_PRESENT))) + break; /* malformed chained buffer */ + + vlib_buffer_t *next_dst_b = vlib_get_buffer (vm, dst_b->next_buffer); + + if (PREDICT_TRUE (1 == next_dst_b->ref_count)) + { + /* normal case: buffer is not cloned, just use it */ + dst_b = next_dst_b; + } + else + { + /* cloned buffer, build a new dest chain from there */ + vlib_buffer_t *bufs[VLIB_BUFFER_LINEARIZE_MAX]; + u32 bis[VLIB_BUFFER_LINEARIZE_MAX + 1]; + const int n = (rem_len + data_size - 1) / data_size; + int n_alloc; + int i; + + ASSERT (n <= VLIB_BUFFER_LINEARIZE_MAX); + if (PREDICT_FALSE (n > VLIB_BUFFER_LINEARIZE_MAX)) + return 0; + + n_alloc = vlib_buffer_alloc (vm, bis, n); + if (PREDICT_FALSE (n_alloc != n)) + { + vlib_buffer_free (vm, bis, n_alloc); + return 0; + } + + vlib_get_buffers (vm, bis, bufs, n); + + for (i = 0; i < n - 1; i++) + { + bufs[i]->flags |= VLIB_BUFFER_NEXT_PRESENT; + bufs[i]->next_buffer = bis[i + 1]; + } + + to_free = dst_b->next_buffer; + dst_b->next_buffer = bis[0]; + dst_b = bufs[0]; + } + + n_buffers++; + + dst_b->current_data = clib_min (0, dst_b->current_data); + dst_b->current_length = 0; + + dst = dst_b->data + dst_b->current_data; + dst_len = data_size - dst_b->current_data; + } - clib_mem_set_heap (oldheap); - clib_warning ("%.6f buffer %llx (%d): %s, not %s", - vlib_time_now (vm), bi, - p[0] ? "busy" : "free", expected ? "busy" : "free"); - os_panic (); - } -out: - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif -} + copy_len = clib_min (src_len, dst_len); -static inline void -vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - void *oldheap; + if (PREDICT_TRUE (src == dst)) + { + /* nothing to do */ + } + else if (src + copy_len > dst && dst + copy_len > src) + { + /* src and dst overlap */ + ASSERT (b == dst_b); + memmove (dst, src, copy_len); + } + else + { + clib_memcpy_fast (dst, src, copy_len); + } + + dst_b->current_length += copy_len; - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); + dst += copy_len; + src += copy_len; + dst_len -= copy_len; + src_len -= copy_len; + rem_len -= copy_len; + } - while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1)) - ; + /* in case of a malformed chain buffer, we'll exit early from the loop. */ + ASSERT (0 == rem_len); + b->total_length_not_including_first_buffer -= rem_len; - hash_set (vlib_buffer_state_validation_hash, b, expected); + if (to_free) + vlib_buffer_free_one (vm, to_free); - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif + if (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT) + { + /* the resulting chain is smaller than the original, cut it there */ + dst_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT; + vlib_buffer_free_one (vm, dst_b->next_buffer); + if (1 == n_buffers) + { + /* no longer a chained buffer */ + dst_b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; + dst_b->total_length_not_including_first_buffer = 0; + } + } + + return n_buffers; } #endif /* included_vlib_buffer_funcs_h */