X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer_funcs.h;h=ec0c12e3ee95dd06ba6e0ba4a6fa260d8a2fe402;hb=5a6c8096cb51065b056c927133781c9f2db9148d;hp=54fc1f61598701a046fef882d5f6b4bcc1dfc33c;hpb=671e60e65635b8d030bf303c88411192c747b59e;p=vpp.git diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 54fc1f61598..ec0c12e3ee9 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -47,6 +47,29 @@ vlib buffer access methods. */ +always_inline void +vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; + + /* reference count in allocated buffer always must be 1 or higher */ + ASSERT (b->ref_count > 0); + + /* verify that buffer pool index is valid */ + bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index); + ASSERT (pointer_to_uword (b) >= bp->start); + ASSERT (pointer_to_uword (b) < bp->start + bp->size - + (bp->data_size + sizeof (vlib_buffer_t))); +} + +always_inline void * +vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index, + uword offset) +{ + offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; + return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *); +} /** \brief Translate buffer index into buffer pointer @@ -57,11 +80,50 @@ always_inline vlib_buffer_t * vlib_get_buffer (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_main_t *bm = &buffer_main; - uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; - ASSERT (offset < bm->buffer_mem_size); + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_t *b; - return uword_to_pointer (bm->buffer_mem_start + offset, void *); + b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0); + vlib_buffer_validate (vm, b); + return b; +} + +static_always_inline u32 +vlib_buffer_get_default_data_size (vlib_main_t * vm) +{ + return vm->buffer_main->default_data_size; +} + +static_always_inline void +vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices) +{ + clib_memcpy_fast (dst, src, n_indices * sizeof (u32)); +} + +STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); +static_always_inline void +vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt) +{ +#if defined CLIB_HAVE_VEC512 + b->as_u8x64[0] = bt->as_u8x64[0]; +#elif defined (CLIB_HAVE_VEC256) + b->as_u8x32[0] = bt->as_u8x32[0]; + b->as_u8x32[1] = bt->as_u8x32[1]; +#elif defined (CLIB_HAVE_VEC128) + b->as_u8x16[0] = bt->as_u8x16[0]; + b->as_u8x16[1] = bt->as_u8x16[1]; + b->as_u8x16[2] = bt->as_u8x16[2]; + b->as_u8x16[3] = bt->as_u8x16[3]; +#else + clib_memcpy_fast (b, bt, 64); +#endif +} + +always_inline u8 +vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node) +{ + ASSERT (numa_node < vm->buffer_main->n_numa_nodes); + return numa_node; } /** \brief Translate array of buffer indices into buffer pointers with offset @@ -76,8 +138,9 @@ static_always_inline void vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, i32 offset) { + uword buffer_mem_start = vm->buffer_main->buffer_mem_start; #ifdef CLIB_HAVE_VEC256 - u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset); + u64x4 off = u64x4_splat (buffer_mem_start + offset); /* if count is not const, compiler will not unroll while loop se we maintain two-in-parallel variant */ while (count >= 8) @@ -99,7 +162,7 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, /* shift and add to get vlib_buffer_t pointer */ u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); #elif defined (CLIB_HAVE_VEC128) - u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset); + u64x2 off = u64x2_splat (buffer_mem_start + offset); u32x4 bi4 = u32x4_load_unaligned (bi); u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4); #if defined (__aarch64__) @@ -111,10 +174,10 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2); #else - b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset; - b[1] = ((u8 *) vlib_get_buffer (vm, bi[1])) + offset; - b[2] = ((u8 *) vlib_get_buffer (vm, bi[2])) + offset; - b[3] = ((u8 *) vlib_get_buffer (vm, bi[3])) + offset; + b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset); + b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset); + b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset); + b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset); #endif b += 4; bi += 4; @@ -122,7 +185,7 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, } while (count) { - b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset; + b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset); b += 1; bi += 1; count -= 1; @@ -153,7 +216,7 @@ vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count) always_inline u32 vlib_get_buffer_index (vlib_main_t * vm, void *p) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; uword offset = pointer_to_uword (p) - bm->buffer_mem_start; ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start); ASSERT (offset < bm->buffer_mem_size); @@ -175,7 +238,7 @@ vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi, { #ifdef CLIB_HAVE_VEC256 u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 }; - u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset); + u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset); while (count >= 8) { @@ -356,9 +419,9 @@ void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, expected_state); always_inline vlib_buffer_known_state_t -vlib_buffer_is_known (u32 buffer_index) +vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; clib_spinlock_lock (&bm->buffer_known_hash_lockp); uword *p = hash_get (bm->buffer_known_hash, buffer_index); @@ -366,51 +429,48 @@ vlib_buffer_is_known (u32 buffer_index) return p ? p[0] : VLIB_BUFFER_UNKNOWN; } -always_inline void -vlib_buffer_set_known_state (u32 buffer_index, - vlib_buffer_known_state_t state) -{ - vlib_buffer_main_t *bm = &buffer_main; - - clib_spinlock_lock (&bm->buffer_known_hash_lockp); - hash_set (bm->buffer_known_hash, buffer_index, state); - clib_spinlock_unlock (&bm->buffer_known_hash_lockp); -} - /* Validates sanity of a single buffer. Returns format'ed vector with error message if any. */ u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -always_inline u32 -vlib_buffer_round_size (u32 size) +static_always_inline vlib_buffer_pool_t * +vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) { - return round_pow2 (size, sizeof (vlib_buffer_t)); + vlib_buffer_main_t *bm = vm->buffer_main; + return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); } -always_inline vlib_buffer_free_list_index_t -vlib_buffer_get_free_list_index (vlib_buffer_t * b) +static_always_inline uword +vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, + u32 n_buffers) { - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NON_DEFAULT_FREELIST)) - return b->free_list_index; + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + u32 len; - return 0; -} + ASSERT (bp->buffers); -always_inline void -vlib_buffer_set_free_list_index (vlib_buffer_t * b, - vlib_buffer_free_list_index_t index) -{ - if (PREDICT_FALSE (index)) + clib_spinlock_lock (&bp->lock); + len = vec_len (bp->buffers); + if (PREDICT_TRUE (n_buffers < len)) { - b->flags |= VLIB_BUFFER_NON_DEFAULT_FREELIST; - b->free_list_index = index; + len -= n_buffers; + vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers); + _vec_len (bp->buffers) = len; + clib_spinlock_unlock (&bp->lock); + return n_buffers; } else - b->flags &= ~VLIB_BUFFER_NON_DEFAULT_FREELIST; + { + vlib_buffer_copy_indices (buffers, bp->buffers, len); + _vec_len (bp->buffers) = 0; + clib_spinlock_unlock (&bp->lock); + return len; + } } -/** \brief Allocate buffers from specific freelist into supplied array + +/** \brief Allocate buffers from specific pool into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @@ -418,57 +478,87 @@ vlib_buffer_set_free_list_index (vlib_buffer_t * b, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ + always_inline u32 -vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, - vlib_buffer_free_list_index_t index) +vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u8 buffer_pool_index) { - vlib_buffer_main_t *bm = &buffer_main; - vlib_buffer_free_list_t *fl; - u32 *src; - uword len; - - ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; + vlib_buffer_pool_thread_t *bpt; + u32 *src, *dst, len, n_left; - fl = pool_elt_at_index (vm->buffer_free_list_pool, index); + bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index); + bpt = vec_elt_at_index (bp->threads, vm->thread_index); - len = vec_len (fl->buffers); + dst = buffers; + n_left = n_buffers; + len = vec_len (bpt->cached_buffers); - if (PREDICT_FALSE (len < n_buffers)) + /* per-thread cache contains enough buffers */ + if (len >= n_buffers) { - bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); - if (PREDICT_FALSE ((len = vec_len (fl->buffers)) == 0)) - return 0; + src = bpt->cached_buffers + len - n_buffers; + vlib_buffer_copy_indices (dst, src, n_buffers); + _vec_len (bpt->cached_buffers) -= n_buffers; - /* even if fill free list didn't manage to refill free list - we should give what we have */ - n_buffers = clib_min (len, n_buffers); + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + return n_buffers; + } - /* following code is intentionaly duplicated to allow compiler - to optimize fast path when n_buffers is constant value */ - src = fl->buffers + len - n_buffers; - clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); - _vec_len (fl->buffers) -= n_buffers; + /* take everything available in the cache */ + if (len) + { + vlib_buffer_copy_indices (dst, bpt->cached_buffers, len); + _vec_len (bpt->cached_buffers) = 0; + dst += len; + n_left -= len; + } - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); + len = round_pow2 (n_left, 32); + vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES); + len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers, + len); + _vec_len (bpt->cached_buffers) = len; - return n_buffers; + if (len) + { + u32 n_copy = clib_min (len, n_left); + src = bpt->cached_buffers + len - n_copy; + vlib_buffer_copy_indices (dst, src, n_copy); + _vec_len (bpt->cached_buffers) -= n_copy; + n_left -= n_copy; } - src = fl->buffers + len - n_buffers; - clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); - _vec_len (fl->buffers) -= n_buffers; + n_buffers -= n_left; /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); return n_buffers; } +/** \brief Allocate buffers from specific numa node into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @param numa_node - (u32) numa node + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u32 numa_node) +{ + u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node); + return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index); +} + /** \brief Allocate buffers into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @@ -477,11 +567,11 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ + always_inline u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { - return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node); } /** \brief Allocate buffers into ring @@ -513,6 +603,216 @@ vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, return n_alloc; } +/** \brief Allocate buffers into ring from specific buffer pool + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers, + u8 buffer_pool_index) +{ + u32 n_alloc; + + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers, + buffer_pool_index); + + n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start, + buffer_pool_index); + + if (PREDICT_TRUE (n_alloc == ring_size - start)) + n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc, + buffer_pool_index); + + return n_alloc; +} + +static_always_inline void +vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index, + u32 * buffers, u32 n_buffers) +{ + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_pool_thread_t *bpt = + vec_elt_at_index (bp->threads, vm->thread_index); + + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_ALLOCATED); + + vec_add_aligned (bpt->cached_buffers, buffers, n_buffers, + CLIB_CACHE_LINE_BYTES); + + if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE) + { + clib_spinlock_lock (&bp->lock); + /* keep last stored buffers, as they are more likely hot in the cache */ + vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0); + bpt->n_alloc -= VLIB_FRAME_SIZE; + clib_spinlock_unlock (&bp->lock); + } +} + +static_always_inline void +vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + int maybe_next) +{ + const int queue_size = 128; + vlib_buffer_pool_t *bp = 0; + u8 buffer_pool_index = ~0; + u32 n_queue = 0, queue[queue_size + 4]; + vlib_buffer_t bt = { }; +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; + vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 }; + vlib_buffer_t flags_refs_mask = { + .flags = VLIB_BUFFER_NEXT_PRESENT, + .ref_count = ~0 + }; +#endif + + while (n_buffers) + { + vlib_buffer_t *b[8]; + u32 bi, sum = 0, flags, next; + + if (n_buffers < 12) + goto one_by_one; + + vlib_get_buffers (vm, buffers, b, 4); + vlib_get_buffers (vm, buffers + 8, b + 4, 4); + + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + u8x16 p0, p1, p2, p3, r; + p0 = u8x16_load_unaligned (b[0]); + p1 = u8x16_load_unaligned (b[1]); + p2 = u8x16_load_unaligned (b[2]); + p3 = u8x16_load_unaligned (b[3]); + + r = p0 ^ bpi_vec.as_u8x16[0]; + r |= p1 ^ bpi_vec.as_u8x16[0]; + r |= p2 ^ bpi_vec.as_u8x16[0]; + r |= p3 ^ bpi_vec.as_u8x16[0]; + r &= bpi_mask.as_u8x16[0]; + r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0]; + + sum = !u8x16_is_all_zero (r); +#else + sum |= b[0]->flags; + sum |= b[1]->flags; + sum |= b[2]->flags; + sum |= b[3]->flags; + sum &= VLIB_BUFFER_NEXT_PRESENT; + sum += b[0]->ref_count - 1; + sum += b[1]->ref_count - 1; + sum += b[2]->ref_count - 1; + sum += b[3]->ref_count - 1; + sum |= b[0]->buffer_pool_index ^ buffer_pool_index; + sum |= b[1]->buffer_pool_index ^ buffer_pool_index; + sum |= b[2]->buffer_pool_index ^ buffer_pool_index; + sum |= b[3]->buffer_pool_index ^ buffer_pool_index; +#endif + + if (sum) + goto one_by_one; + + vlib_buffer_copy_indices (queue + n_queue, buffers, 4); + vlib_buffer_copy_template (b[0], &bt); + vlib_buffer_copy_template (b[1], &bt); + vlib_buffer_copy_template (b[2], &bt); + vlib_buffer_copy_template (b[3], &bt); + n_queue += 4; + + vlib_buffer_validate (vm, b[0]); + vlib_buffer_validate (vm, b[1]); + vlib_buffer_validate (vm, b[2]); + vlib_buffer_validate (vm, b[3]); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); + + if (n_queue >= queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + buffers += 4; + n_buffers -= 4; + continue; + + one_by_one: + bi = buffers[0]; + + next_in_chain: + b[0] = vlib_get_buffer (vm, bi); + flags = b[0]->flags; + next = b[0]->next_buffer; + + if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index)) + { + + if (n_queue) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + + buffer_pool_index = b[0]->buffer_pool_index; +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + bpi_vec.buffer_pool_index = buffer_pool_index; +#endif + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); + } + + vlib_buffer_validate (vm, b[0]); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + + if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0) + { + vlib_buffer_copy_template (b[0], &bt); + queue[n_queue++] = bi; + } + + if (n_queue == queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size); + n_queue = 0; + } + + if (flags & VLIB_BUFFER_NEXT_PRESENT) + { + bi = next; + goto next_in_chain; + } + + buffers++; + n_buffers--; + } + + if (n_queue) + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); +} + + /** \brief Free buffers Frees the entire buffer chain for each buffer @@ -528,11 +828,7 @@ vlib_buffer_free (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = &buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_cb); - - return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1); } /** \brief Free buffers, does not free the buffer chain for each buffer @@ -549,11 +845,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = &buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_no_next_cb); - - return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0); } /** \brief Free one buffer @@ -565,7 +857,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, always_inline void vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1); + vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1); } /** \brief Free buffers from ring @@ -618,63 +910,9 @@ vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start, } } -/* Add/delete buffer free lists. */ -vlib_buffer_free_list_index_t vlib_buffer_create_free_list (vlib_main_t * vm, - u32 n_data_bytes, - char *fmt, ...); -always_inline void -vlib_buffer_delete_free_list (vlib_main_t * vm, - vlib_buffer_free_list_index_t free_list_index) -{ - vlib_buffer_main_t *bm = &buffer_main; - - ASSERT (bm->cb.vlib_buffer_delete_free_list_cb); - - bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index); -} - -/* Make sure we have at least given number of unaligned buffers. */ -void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm, - vlib_buffer_free_list_t * - free_list, - uword n_unaligned_buffers); - -always_inline vlib_buffer_free_list_t * -vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b, - vlib_buffer_free_list_index_t * index) -{ - vlib_buffer_free_list_index_t i; - - *index = i = vlib_buffer_get_free_list_index (b); - return pool_elt_at_index (vm->buffer_free_list_pool, i); -} - -always_inline vlib_buffer_free_list_t * -vlib_buffer_get_free_list (vlib_main_t * vm, - vlib_buffer_free_list_index_t free_list_index) -{ - vlib_buffer_free_list_t *f; - - f = pool_elt_at_index (vm->buffer_free_list_pool, free_list_index); - - /* Sanity: indices must match. */ - ASSERT (f->index == free_list_index); - - return f; -} - -always_inline u32 -vlib_buffer_free_list_buffer_size (vlib_main_t * vm, - vlib_buffer_free_list_index_t index) -{ - vlib_buffer_free_list_t *f = vlib_buffer_get_free_list (vm, index); - return f->n_data_bytes; -} - /* Append given data to end of buffer, possibly allocating new buffers. */ -u32 vlib_buffer_add_data (vlib_main_t * vm, - vlib_buffer_free_list_index_t free_list_index, - u32 buffer_index, void *data, u32 n_data_bytes); +int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, + u32 n_data_bytes); /* duplicate all buffers in chain */ always_inline vlib_buffer_t * @@ -753,7 +991,7 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, u16 i; vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); - ASSERT (s->n_add_refs == 0); + ASSERT (s->ref_count == 1); ASSERT (n_buffers); ASSERT (n_buffers <= 256); @@ -778,17 +1016,15 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, return 1; } - n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - vlib_buffer_get_free_list_index - (s)); + n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, + s->buffer_pool_index); for (i = 0; i < n_buffers; i++) { vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); d->current_data = s->current_data; d->current_length = head_end_offset; - vlib_buffer_set_free_list_index (d, - vlib_buffer_get_free_list_index (s)); + ASSERT (d->buffer_pool_index == s->buffer_pool_index); d->total_length_not_including_first_buffer = s->current_length - head_end_offset; @@ -806,11 +1042,11 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { s = vlib_get_buffer (vm, s->next_buffer); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers; } return n_buffers; @@ -864,8 +1100,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT (vlib_buffer_get_free_list_index (head) == - vlib_buffer_get_free_list_index (tail)); + ASSERT (head->buffer_pool_index == tail->buffer_pool_index); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -876,7 +1111,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, tail->total_length_not_including_first_buffer; next_segment: - clib_atomic_add_fetch (&tail->n_add_refs, 1); + clib_atomic_add_fetch (&tail->ref_count, 1); if (tail->flags & VLIB_BUFFER_NEXT_PRESENT) { @@ -925,12 +1160,10 @@ vlib_buffer_chain_increase_length (vlib_buffer_t * first, * Returns the number of copied bytes. */ always_inline u16 vlib_buffer_chain_append_data (vlib_main_t * vm, - vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t * first, vlib_buffer_t * last, void *data, u16 data_len) { - u32 n_buffer_bytes = - vlib_buffer_free_list_buffer_size (vm, free_list_index); + u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); ASSERT (n_buffer_bytes >= last->current_length + last->current_data); u16 len = clib_min (data_len, n_buffer_bytes - last->current_length - @@ -948,8 +1181,6 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, * chained and points to the last buffer in the chain. */ u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - vlib_buffer_free_list_index_t - free_list_index, vlib_buffer_t * first, vlib_buffer_t ** last, void *data, u16 data_len); @@ -966,17 +1197,9 @@ typedef struct /* Number of buffers to allocate in each call to allocator. */ u32 min_n_buffers_each_alloc; - /* Buffer free list for this template. */ - vlib_buffer_free_list_index_t free_list_index; - - u32 *free_buffers; - u8 *name; } vlib_packet_template_t; -void vlib_packet_template_get_packet_helper (vlib_main_t * vm, - vlib_packet_template_t * t); - void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, void *packet_data, @@ -994,148 +1217,9 @@ vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t) vec_free (t->packet_data); } -/* Set a buffer quickly into "uninitialized" state. We want this to - be extremely cheap and arrange for all fields that need to be - initialized to be in the first 128 bits of the buffer. */ -always_inline void -vlib_buffer_init_for_free_list (vlib_buffer_t * dst, - vlib_buffer_free_list_t * fl) -{ - vlib_buffer_t *src = &fl->buffer_init_template; - - /* Make sure vlib_buffer_t is cacheline aligned and sized */ - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0); - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) == - CLIB_CACHE_LINE_BYTES); - ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) == - CLIB_CACHE_LINE_BYTES * 2); - - /* Make sure buffer template is sane. */ - ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - - clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - /* Not in the first 16 octets. */ - dst->n_add_refs = src->n_add_refs; - vlib_buffer_set_free_list_index (dst, fl->index); - - /* Make sure it really worked. */ -#define _(f) ASSERT (dst->f == src->f); - _(current_data); - _(current_length); - _(flags); -#undef _ - /* ASSERT (dst->total_length_not_including_first_buffer == 0); */ - /* total_length_not_including_first_buffer is not in the template anymore - * so it may actually not zeroed for some buffers. One option is to - * uncomment the line lower (comes at a cost), the other, is to just not - * care */ - /* dst->total_length_not_including_first_buffer = 0; */ - ASSERT (dst->n_add_refs == 0); -} - -always_inline void -vlib_buffer_add_to_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * f, - u32 buffer_index, u8 do_init) -{ - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index); - vlib_buffer_t *b; - b = vlib_get_buffer (vm, buffer_index); - if (PREDICT_TRUE (do_init)) - vlib_buffer_init_for_free_list (b, f); - vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES); - - if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE) - { - clib_spinlock_lock (&bp->lock); - /* keep last stored buffers, as they are more likely hot in the cache */ - vec_add_aligned (bp->buffers, f->buffers, VLIB_FRAME_SIZE, - CLIB_CACHE_LINE_BYTES); - vec_delete (f->buffers, VLIB_FRAME_SIZE, 0); - f->n_alloc -= VLIB_FRAME_SIZE; - clib_spinlock_unlock (&bp->lock); - } -} - -#if CLIB_DEBUG > 0 -extern u32 *vlib_buffer_state_validation_lock; -extern uword *vlib_buffer_state_validation_hash; -extern void *vlib_buffer_state_heap; -#endif - -static inline void -vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - uword *p; - void *oldheap; - - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); - - while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock)) - ; - - p = hash_get (vlib_buffer_state_validation_hash, b); - - /* If we don't know about b, declare it to be in the expected state */ - if (!p) - { - hash_set (vlib_buffer_state_validation_hash, b, expected); - goto out; - } - - if (p[0] != expected) - { - void cj_stop (void); - u32 bi; - vlib_main_t *vm = &vlib_global_main; - - cj_stop (); - - bi = vlib_get_buffer_index (vm, b); - - clib_mem_set_heap (oldheap); - clib_warning ("%.6f buffer %llx (%d): %s, not %s", - vlib_time_now (vm), bi, - p[0] ? "busy" : "free", expected ? "busy" : "free"); - os_panic (); - } -out: - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif -} - -static inline void -vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - void *oldheap; - - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); - - while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock)) - ; - - hash_set (vlib_buffer_state_validation_hash, b, expected); - - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif -} - -/** minimum data size of first buffer in a buffer chain */ -#define VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE (256) - /** * @brief compress buffer chain in a way where the first buffer is at least - * VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE long + * VLIB_BUFFER_CLONE_HEAD_SIZE long * * @param[in] vm - vlib_main * @param[in,out] first - first buffer in chain @@ -1146,19 +1230,15 @@ always_inline void vlib_buffer_chain_compress (vlib_main_t * vm, vlib_buffer_t * first, u32 ** discard_vector) { - if (first->current_length >= VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE || + if (first->current_length >= VLIB_BUFFER_CLONE_HEAD_SIZE || !(first->flags & VLIB_BUFFER_NEXT_PRESENT)) { /* this is already big enough or not a chain */ return; } - /* probe free list to find allocated buffer size to avoid overfill */ - vlib_buffer_free_list_index_t index; - vlib_buffer_free_list_t *free_list = - vlib_buffer_get_buffer_free_list (vm, first, &index); - u32 want_first_size = clib_min (VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE, - free_list->n_data_bytes - + u32 want_first_size = clib_min (VLIB_BUFFER_CLONE_HEAD_SIZE, + vlib_buffer_get_default_data_size (vm) - first->current_data); do { @@ -1205,9 +1285,7 @@ always_inline int vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * first) { vlib_buffer_t *b = first; - vlib_buffer_free_list_t *fl = - vlib_buffer_get_free_list (vm, vlib_buffer_get_free_list_index (b)); - u32 buf_len = fl->n_data_bytes; + u32 buf_len = vlib_buffer_get_default_data_size (vm); // free buffer chain starting from the second buffer int free_count = (b->flags & VLIB_BUFFER_NEXT_PRESENT) != 0; u32 chain_to_free = b->next_buffer;