X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=src%2Fvlib%2Fbuffer_funcs.h;h=89a765ee0d35e12d90533364eba56f544f50f732;hb=1a19552eee24e447e6087de43a2eeb9250b8cae7;hp=483a990731fa8e912b6933f85d8f1620efadb00a;hpb=da7f7b6164e976a97ff0afb13f488c60461402bc;p=vpp.git diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 483a990731f..89a765ee0d3 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -51,6 +51,32 @@ vlib buffer access methods. */ +typedef void (vlib_buffer_enqueue_to_next_fn_t) (vlib_main_t *vm, + vlib_node_runtime_t *node, + u32 *buffers, u16 *nexts, + uword count); +typedef void (vlib_buffer_enqueue_to_single_next_fn_t) ( + vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index, + u32 count); + +typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) ( + vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, + u32 *buffer_indices, u16 *thread_indices, u32 n_packets, + int drop_on_congestion); + +typedef u32 (vlib_frame_queue_dequeue_fn_t) (vlib_main_t *vm, + vlib_frame_queue_main_t *fqm); + +typedef struct +{ + vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn; + vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn; + vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn; + vlib_frame_queue_dequeue_fn_t *frame_queue_dequeue_fn; +} vlib_buffer_func_main_t; + +extern vlib_buffer_func_main_t vlib_buffer_func_main; + always_inline void vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b) { @@ -101,7 +127,43 @@ vlib_buffer_get_default_data_size (vlib_main_t * vm) static_always_inline void vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices) { - clib_memcpy_fast (dst, src, n_indices * sizeof (u32)); + clib_memcpy_u32 (dst, src, n_indices); +} + +always_inline void +vlib_buffer_copy_indices_from_ring (u32 * dst, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers) +{ + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + { + vlib_buffer_copy_indices (dst, ring + start, n_buffers); + } + else + { + u32 n = ring_size - start; + vlib_buffer_copy_indices (dst, ring + start, n); + vlib_buffer_copy_indices (dst + n, ring, n_buffers - n); + } +} + +always_inline void +vlib_buffer_copy_indices_to_ring (u32 * ring, u32 * src, u32 start, + u32 ring_size, u32 n_buffers) +{ + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + { + vlib_buffer_copy_indices (ring + start, src, n_buffers); + } + else + { + u32 n = ring_size - start; + vlib_buffer_copy_indices (ring + start, src, n); + vlib_buffer_copy_indices (ring, src + n, n_buffers - n); + } } STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); @@ -143,37 +205,78 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, i32 offset) { uword buffer_mem_start = vm->buffer_main->buffer_mem_start; -#ifdef CLIB_HAVE_VEC256 - u64x4 off = u64x4_splat (buffer_mem_start + offset); +#ifdef CLIB_HAVE_VEC512 + u64x8 of8 = u64x8_splat (buffer_mem_start + offset); + u64x4 off = u64x8_extract_lo (of8); /* if count is not const, compiler will not unroll while loop se we maintain two-in-parallel variant */ + while (count >= 32) + { + u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi)); + u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8)); + u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16)); + u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24)); + /* shift and add to get vlib_buffer_t pointer */ + u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b); + u64x8_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 8); + u64x8_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 16); + u64x8_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 24); + b += 32; + bi += 32; + count -= 32; + } while (count >= 8) { - u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi)); - u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4)); + u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi)); /* shift and add to get vlib_buffer_t pointer */ - u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); - u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4); + u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b); b += 8; bi += 8; count -= 8; } +#elif defined CLIB_HAVE_VEC256 + u64x4 off = u64x4_splat (buffer_mem_start + offset); + /* if count is not const, compiler will not unroll while loop + se we maintain two-in-parallel variant */ + while (count >= 32) + { + u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi)); + u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4)); + u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8)); + u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12)); + u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16)); + u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20)); + u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24)); + u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28)); + /* shift and add to get vlib_buffer_t pointer */ + u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); + u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4); + u64x4_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 8); + u64x4_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 12); + u64x4_store_unaligned ((b4 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 16); + u64x4_store_unaligned ((b5 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 20); + u64x4_store_unaligned ((b6 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 24); + u64x4_store_unaligned ((b7 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 28); + b += 32; + bi += 32; + count -= 32; + } #endif while (count >= 4) { #ifdef CLIB_HAVE_VEC256 - u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi)); + u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi)); /* shift and add to get vlib_buffer_t pointer */ u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); #elif defined (CLIB_HAVE_VEC128) u64x2 off = u64x2_splat (buffer_mem_start + offset); u32x4 bi4 = u32x4_load_unaligned (bi); - u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4); + u64x2 b0 = u64x2_from_u32x4 ((u32x4) bi4); #if defined (__aarch64__) - u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4); + u64x2 b1 = u64x2_from_u32x4_high ((u32x4) bi4); #else bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1); - u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4); + u64x2 b1 = u64x2_from_u32x4 ((u32x4) bi4); #endif u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2); @@ -438,6 +541,13 @@ vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); +u8 *vlib_validate_buffers (vlib_main_t * vm, + u32 * buffers, + uword next_buffer_stride, + uword n_buffers, + vlib_buffer_known_state_t known_state, + uword follow_buffer_next); + static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) { @@ -445,7 +555,7 @@ vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); } -static_always_inline uword +static_always_inline __clib_warn_unused_result uword vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, u32 n_buffers) { @@ -455,19 +565,19 @@ vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, ASSERT (bp->buffers); clib_spinlock_lock (&bp->lock); - len = vec_len (bp->buffers); + len = bp->n_avail; if (PREDICT_TRUE (n_buffers < len)) { len -= n_buffers; vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers); - _vec_len (bp->buffers) = len; + bp->n_avail = len; clib_spinlock_unlock (&bp->lock); return n_buffers; } else { vlib_buffer_copy_indices (buffers, bp->buffers, len); - _vec_len (bp->buffers) = 0; + bp->n_avail = 0; clib_spinlock_unlock (&bp->lock); return len; } @@ -483,7 +593,7 @@ vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, less than the number requested or zero */ -always_inline u32 +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u8 buffer_pool_index) { @@ -492,19 +602,42 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, vlib_buffer_pool_thread_t *bpt; u32 *src, *dst, len, n_left; + /* If buffer allocation fault injection is configured */ + if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0) + { + u32 vlib_buffer_alloc_may_fail (vlib_main_t *, u32); + + /* See how many buffers we're willing to allocate */ + n_buffers = vlib_buffer_alloc_may_fail (vm, n_buffers); + if (n_buffers == 0) + return (n_buffers); + } + bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index); bpt = vec_elt_at_index (bp->threads, vm->thread_index); dst = buffers; n_left = n_buffers; - len = vec_len (bpt->cached_buffers); + len = bpt->n_cached; /* per-thread cache contains enough buffers */ if (len >= n_buffers) { src = bpt->cached_buffers + len - n_buffers; vlib_buffer_copy_indices (dst, src, n_buffers); - _vec_len (bpt->cached_buffers) -= n_buffers; + bpt->n_cached -= n_buffers; + + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + return n_buffers; + } + + /* alloc bigger than cache - take buffers directly from main pool */ + if (n_buffers >= VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ) + { + n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers, + n_buffers); if (CLIB_DEBUG > 0) vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, @@ -516,23 +649,22 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, if (len) { vlib_buffer_copy_indices (dst, bpt->cached_buffers, len); - _vec_len (bpt->cached_buffers) = 0; + bpt->n_cached = 0; dst += len; n_left -= len; } len = round_pow2 (n_left, 32); - vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES); len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers, len); - _vec_len (bpt->cached_buffers) = len; + bpt->n_cached = len; if (len) { u32 n_copy = clib_min (len, n_left); src = bpt->cached_buffers + len - n_copy; vlib_buffer_copy_indices (dst, src, n_copy); - _vec_len (bpt->cached_buffers) -= n_copy; + bpt->n_cached -= n_copy; n_left -= n_copy; } @@ -555,7 +687,7 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -always_inline u32 +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 numa_node) { @@ -572,7 +704,7 @@ vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, less than the number requested or zero */ -always_inline u32 +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node); @@ -588,7 +720,7 @@ vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -always_inline u32 +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers) { @@ -617,7 +749,7 @@ vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -always_inline u32 +always_inline __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index) @@ -645,26 +777,33 @@ vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, u32 n_buffers) { vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); - vlib_buffer_pool_thread_t *bpt = - vec_elt_at_index (bp->threads, vm->thread_index); + vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads, + vm->thread_index); + u32 n_cached, n_empty; if (CLIB_DEBUG > 0) vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, VLIB_BUFFER_KNOWN_ALLOCATED); - vec_add_aligned (bpt->cached_buffers, buffers, n_buffers, - CLIB_CACHE_LINE_BYTES); - - if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE) + n_cached = bpt->n_cached; + n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached; + if (n_buffers <= n_empty) { - clib_spinlock_lock (&bp->lock); - /* keep last stored buffers, as they are more likely hot in the cache */ - vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE, - CLIB_CACHE_LINE_BYTES); - vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0); - bpt->n_alloc -= VLIB_FRAME_SIZE; - clib_spinlock_unlock (&bp->lock); + vlib_buffer_copy_indices (bpt->cached_buffers + n_cached, + buffers, n_buffers); + bpt->n_cached = n_cached + n_buffers; + return; } + + vlib_buffer_copy_indices (bpt->cached_buffers + n_cached, + buffers + n_buffers - n_empty, n_empty); + bpt->n_cached = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ; + + clib_spinlock_lock (&bp->lock); + vlib_buffer_copy_indices (bp->buffers + bp->n_avail, buffers, + n_buffers - n_empty); + bp->n_avail += n_buffers - n_empty; + clib_spinlock_unlock (&bp->lock); } static_always_inline void @@ -676,32 +815,46 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u8 buffer_pool_index = ~0; u32 n_queue = 0, queue[queue_size + 4]; vlib_buffer_t bt = { }; -#if defined(CLIB_HAVE_VEC128) && !__aarch64__ +#if defined(CLIB_HAVE_VEC128) vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; - vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 }; + vlib_buffer_t bpi_vec = {}; vlib_buffer_t flags_refs_mask = { .flags = VLIB_BUFFER_NEXT_PRESENT, - .ref_count = ~0 + .ref_count = ~1 }; #endif + if (PREDICT_FALSE (n_buffers == 0)) + return; + + vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]); + buffer_pool_index = b->buffer_pool_index; + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); +#if defined(CLIB_HAVE_VEC128) + bpi_vec.buffer_pool_index = buffer_pool_index; +#endif + while (n_buffers) { vlib_buffer_t *b[8]; u32 bi, sum = 0, flags, next; - if (n_buffers < 12) + if (n_buffers < 4) goto one_by_one; vlib_get_buffers (vm, buffers, b, 4); - vlib_get_buffers (vm, buffers + 8, b + 4, 4); - vlib_prefetch_buffer_header (b[4], LOAD); - vlib_prefetch_buffer_header (b[5], LOAD); - vlib_prefetch_buffer_header (b[6], LOAD); - vlib_prefetch_buffer_header (b[7], LOAD); + if (n_buffers >= 12) + { + vlib_get_buffers (vm, buffers + 8, b + 4, 4); + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + } -#if defined(CLIB_HAVE_VEC128) && !__aarch64__ +#if defined(CLIB_HAVE_VEC128) u8x16 p0, p1, p2, p3, r; p0 = u8x16_load_unaligned (b[0]); p1 = u8x16_load_unaligned (b[1]); @@ -779,7 +932,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, } buffer_pool_index = b[0]->buffer_pool_index; -#if defined(CLIB_HAVE_VEC128) && !__aarch64__ +#if defined(CLIB_HAVE_VEC128) bpi_vec.buffer_pool_index = buffer_pool_index; #endif bp = vlib_get_buffer_pool (vm, buffer_pool_index); @@ -802,7 +955,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, n_queue = 0; } - if (flags & VLIB_BUFFER_NEXT_PRESENT) + if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT)) { bi = next; goto next_in_chain; @@ -918,13 +1071,18 @@ vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start, int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, u32 n_data_bytes); +/* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */ +#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \ + (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \ + VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL) + /* duplicate all buffers in chain */ always_inline vlib_buffer_t * vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) { vlib_buffer_t *s, *d, *fd; uword n_alloc, n_buffers = 1; - u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID; + u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK; int i; s = b; @@ -951,6 +1109,7 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d->current_data = s->current_data; d->current_length = s->current_length; d->flags = s->flags & flag_mask; + d->trace_handle = s->trace_handle; d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer; clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); @@ -997,6 +1156,31 @@ vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di) return d; } +/* \brief Move packet from current position to offset position in buffer. + Only work for small packet using one buffer with room to fit the move + @param vm - (vlib_main_t *) vlib main data structure pointer + @param b - (vlib_buffer_t *) pointer to buffer + @param offset - (i16) position to move the packet in buffer + */ +always_inline void +vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset) +{ + ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); + ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); + ASSERT (offset + b->current_length < + vlib_buffer_get_default_data_size (vm)); + + u8 *source = vlib_buffer_get_current (b); + b->current_data = offset; + u8 *destination = vlib_buffer_get_current (b); + u16 length = b->current_length; + + if (source + length <= destination) /* no overlap */ + clib_memcpy_fast (destination, source, length); + else + memmove (destination, source, length); +} + /** \brief Create a maximum of 256 clones of buffer and store them in the supplied array @@ -1006,12 +1190,14 @@ vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di) @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends + @param offset - (i16) copy packet head at current position if 0, + else at offset position to change headroom space as specified @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ always_inline u16 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, - u16 n_buffers, u16 head_end_offset) + u16 n_buffers, u16 head_end_offset, i16 offset) { u16 i; vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); @@ -1019,10 +1205,16 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, ASSERT (s->ref_count == 1); ASSERT (n_buffers); ASSERT (n_buffers <= 256); + ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); + ASSERT ((offset + head_end_offset) < + vlib_buffer_get_default_data_size (vm)); if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2) { buffers[0] = src_buffer; + if (offset) + vlib_buffer_move (vm, s, offset); + for (i = 1; i < n_buffers; i++) { vlib_buffer_t *d; @@ -1035,7 +1227,7 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, return n_buffers; } - if (PREDICT_FALSE (n_buffers == 1)) + if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0))) { buffers[0] = src_buffer; return 1; @@ -1047,7 +1239,11 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, for (i = 0; i < n_buffers; i++) { vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); - d->current_data = s->current_data; + if (offset) + d->current_data = offset; + else + d->current_data = s->current_data; + d->current_length = head_end_offset; ASSERT (d->buffer_pool_index == s->buffer_pool_index); @@ -1058,8 +1254,9 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, d->total_length_not_including_first_buffer += s->total_length_not_including_first_buffer; } - d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT; - d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID; + d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) | + VLIB_BUFFER_NEXT_PRESENT; + d->trace_handle = s->trace_handle; clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); clib_memcpy_fast (vlib_buffer_get_current (d), @@ -1067,11 +1264,11 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); - s->ref_count = n_buffers; + s->ref_count = n_buffers ? n_buffers : s->ref_count; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { s = vlib_get_buffer (vm, s->next_buffer); - s->ref_count = n_buffers; + s->ref_count = n_buffers ? n_buffers : s->ref_count; } return n_buffers; @@ -1086,12 +1283,14 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends + @param offset - (i16) copy packet head at current position if 0, + else at offset position to change headroom space as specified @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ always_inline u16 -vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, - u16 n_buffers, u16 head_end_offset) +vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers, + u16 n_buffers, u16 head_end_offset, i16 offset) { vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); u16 n_cloned = 0; @@ -1103,16 +1302,36 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, n_cloned += vlib_buffer_clone_256 (vm, vlib_get_buffer_index (vm, copy), (buffers + n_cloned), - 256, head_end_offset); + 256, head_end_offset, offset); n_buffers -= 256; } n_cloned += vlib_buffer_clone_256 (vm, src_buffer, buffers + n_cloned, - n_buffers, head_end_offset); + n_buffers, head_end_offset, offset); return n_cloned; } +/** \brief Create multiple clones of buffer and store them + in the supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param src_buffer - (u32) source buffer index + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u16) number of buffer clones requested (<=256) + @param head_end_offset - (u16) offset relative to current position + where packet head ends + @return - (u16) number of buffers actually cloned, may be + less than the number requested or zero +*/ +always_inline u16 +vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, + u16 n_buffers, u16 head_end_offset) +{ + return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers, + head_end_offset, 0); +} + /** \brief Attach cloned tail to the buffer @param vm - (vlib_main_t *) vlib main data structure pointer @@ -1212,7 +1431,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first); format_function_t format_vlib_buffer, format_vlib_buffer_and_data, - format_vlib_buffer_contents; + format_vlib_buffer_contents, format_vlib_buffer_no_chain; typedef struct { @@ -1249,133 +1468,148 @@ vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b) ((u8 *) vlib_buffer_get_current (b) + b->current_length); } +#define VLIB_BUFFER_LINEARIZE_MAX 64 + always_inline u32 vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b) { - vlib_buffer_t *db = b, *sb, *first = b; - int is_cloned = 0; - u32 bytes_left = 0, data_size; - u16 src_left, dst_left, n_buffers = 1; - u8 *dp, *sp; - u32 to_free = 0; + vlib_buffer_t *dst_b; + u32 n_buffers = 1, to_free = 0; + u16 rem_len, dst_len, data_size, src_len = 0; + u8 *dst, *src = 0; if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) return 1; + ASSERT (1 == b->ref_count); + if (PREDICT_FALSE (1 != b->ref_count)) + return 0; + data_size = vlib_buffer_get_default_data_size (vm); + rem_len = vlib_buffer_length_in_chain (vm, b) - b->current_length; - dst_left = vlib_buffer_space_left_at_end (vm, b); + dst_b = b; + dst = vlib_buffer_get_tail (dst_b); + dst_len = vlib_buffer_space_left_at_end (vm, dst_b); - while (b->flags & VLIB_BUFFER_NEXT_PRESENT) - { - b = vlib_get_buffer (vm, b->next_buffer); - if (b->ref_count > 1) - is_cloned = 1; - bytes_left += b->current_length; - n_buffers++; - } + b->total_length_not_including_first_buffer -= dst_len; - /* if buffer is cloned, create completely new chain - unless everything fits - * into one buffer */ - if (is_cloned && bytes_left >= dst_left) + while (rem_len > 0) { - u32 len = 0; - u32 space_needed = bytes_left - dst_left; - u32 tail; + u16 copy_len; - if (vlib_buffer_alloc (vm, &tail, 1) == 0) - return 0; + while (0 == src_len) + { + ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT); + if (PREDICT_FALSE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))) + break; /* malformed chained buffer */ - ++n_buffers; - len += data_size; - b = vlib_get_buffer (vm, tail); + b = vlib_get_buffer (vm, b->next_buffer); + src = vlib_buffer_get_current (b); + src_len = b->current_length; + } - while (len < space_needed) + if (0 == dst_len) { - u32 bi; - if (vlib_buffer_alloc (vm, &bi, 1) == 0) + ASSERT (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT); + if (PREDICT_FALSE (!(dst_b->flags & VLIB_BUFFER_NEXT_PRESENT))) + break; /* malformed chained buffer */ + + vlib_buffer_t *next_dst_b = vlib_get_buffer (vm, dst_b->next_buffer); + + if (PREDICT_TRUE (1 == next_dst_b->ref_count)) + { + /* normal case: buffer is not cloned, just use it */ + dst_b = next_dst_b; + } + else { - vlib_buffer_free_one (vm, tail); - return 0; + /* cloned buffer, build a new dest chain from there */ + vlib_buffer_t *bufs[VLIB_BUFFER_LINEARIZE_MAX]; + u32 bis[VLIB_BUFFER_LINEARIZE_MAX + 1]; + const int n = (rem_len + data_size - 1) / data_size; + int n_alloc; + int i; + + ASSERT (n <= VLIB_BUFFER_LINEARIZE_MAX); + if (PREDICT_FALSE (n > VLIB_BUFFER_LINEARIZE_MAX)) + return 0; + + n_alloc = vlib_buffer_alloc (vm, bis, n); + if (PREDICT_FALSE (n_alloc != n)) + { + vlib_buffer_free (vm, bis, n_alloc); + return 0; + } + + vlib_get_buffers (vm, bis, bufs, n); + + for (i = 0; i < n - 1; i++) + { + bufs[i]->flags |= VLIB_BUFFER_NEXT_PRESENT; + bufs[i]->next_buffer = bis[i + 1]; + } + + to_free = dst_b->next_buffer; + dst_b->next_buffer = bis[0]; + dst_b = bufs[0]; } - b->flags = VLIB_BUFFER_NEXT_PRESENT; - b->next_buffer = bi; - b = vlib_get_buffer (vm, bi); - len += data_size; + n_buffers++; - } - sb = vlib_get_buffer (vm, first->next_buffer); - to_free = first->next_buffer; - first->next_buffer = tail; - } - else - sb = vlib_get_buffer (vm, first->next_buffer); - src_left = sb->current_length; - sp = vlib_buffer_get_current (sb); - dp = vlib_buffer_get_tail (db); + dst_b->current_data = clib_min (0, dst_b->current_data); + dst_b->current_length = 0; - while (bytes_left) - { - u16 bytes_to_copy; + dst = dst_b->data + dst_b->current_data; + dst_len = data_size - dst_b->current_data; + } + + copy_len = clib_min (src_len, dst_len); - if (dst_left == 0) + if (PREDICT_TRUE (src == dst)) { - if (db != first) - db->current_data = 0; - db->current_length = dp - (u8 *) vlib_buffer_get_current (db); - ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT); - db = vlib_get_buffer (vm, db->next_buffer); - dst_left = data_size; - dp = db->data; + /* nothing to do */ } - - while (src_left == 0) + else if (src + copy_len > dst && dst + copy_len > src) { - ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT); - sb = vlib_get_buffer (vm, sb->next_buffer); - src_left = sb->current_length; - sp = vlib_buffer_get_current (sb); + /* src and dst overlap */ + ASSERT (b == dst_b); + memmove (dst, src, copy_len); } - - bytes_to_copy = clib_min (dst_left, src_left); - - if (dp != sp) + else { - if (sb == db) - bytes_to_copy = clib_min (bytes_to_copy, sp - dp); - - clib_memcpy_fast (dp, sp, bytes_to_copy); + clib_memcpy_fast (dst, src, copy_len); } - src_left -= bytes_to_copy; - dst_left -= bytes_to_copy; - dp += bytes_to_copy; - sp += bytes_to_copy; - bytes_left -= bytes_to_copy; + dst_b->current_length += copy_len; + + dst += copy_len; + src += copy_len; + dst_len -= copy_len; + src_len -= copy_len; + rem_len -= copy_len; } - if (db != first) - db->current_data = 0; - db->current_length = dp - (u8 *) vlib_buffer_get_current (db); - if (is_cloned && to_free) + /* in case of a malformed chain buffer, we'll exit early from the loop. */ + ASSERT (0 == rem_len); + b->total_length_not_including_first_buffer -= rem_len; + + if (to_free) vlib_buffer_free_one (vm, to_free); - else + + if (dst_b->flags & VLIB_BUFFER_NEXT_PRESENT) { - if (db->flags & VLIB_BUFFER_NEXT_PRESENT) - vlib_buffer_free_one (vm, db->next_buffer); - db->flags &= ~VLIB_BUFFER_NEXT_PRESENT; - b = first; - n_buffers = 1; - while (b->flags & VLIB_BUFFER_NEXT_PRESENT) + /* the resulting chain is smaller than the original, cut it there */ + dst_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT; + vlib_buffer_free_one (vm, dst_b->next_buffer); + if (1 == n_buffers) { - b = vlib_get_buffer (vm, b->next_buffer); - ++n_buffers; + /* no longer a chained buffer */ + dst_b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; + dst_b->total_length_not_including_first_buffer = 0; } } - first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; - return n_buffers; }