X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer_funcs.h;h=1ea3c0829b8f3b9440e1e11377ea6331ebb3feb4;hb=refs%2Fchanges%2F61%2F9861%2F9;hp=394c336ac49e08756be4d62d4e05d7e5d599333b;hpb=9d676afbb779da5186cb3869925ef6d7d3d04db1;p=vpp.git diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 394c336ac49..1ea3c0829b8 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -56,8 +56,11 @@ always_inline vlib_buffer_t * vlib_get_buffer (vlib_main_t * vm, u32 buffer_index) { - return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index) - << CLIB_LOG2_CACHE_LINE_BYTES); + vlib_buffer_main_t *bm = vm->buffer_main; + uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; + ASSERT (offset < bm->buffer_mem_size); + + return uword_to_pointer (bm->buffer_mem_start + offset, void *); } /** \brief Translate buffer pointer into buffer index @@ -66,10 +69,14 @@ vlib_get_buffer (vlib_main_t * vm, u32 buffer_index) @param p - (void *) buffer pointer @return - (u32) buffer index */ + always_inline u32 vlib_get_buffer_index (vlib_main_t * vm, void *p) { - uword offset = vlib_physmem_offset_of (&vm->physmem_main, p); + vlib_buffer_main_t *bm = vm->buffer_main; + uword offset = pointer_to_uword (p) - bm->buffer_mem_start; + ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start); + ASSERT (offset < bm->buffer_mem_size); ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0); return offset >> CLIB_LOG2_CACHE_LINE_BYTES; } @@ -99,12 +106,15 @@ uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, always_inline uword vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b) { - uword l = b->current_length + b->total_length_not_including_first_buffer; - if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT - | VLIB_BUFFER_TOTAL_LENGTH_VALID)) - == VLIB_BUFFER_NEXT_PRESENT)) - return vlib_buffer_length_in_chain_slow_path (vm, b); - return l; + uword len = b->current_length; + + if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + return len; + + if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)) + return len + b->total_length_not_including_first_buffer; + + return vlib_buffer_length_in_chain_slow_path (vm, b); } /** \brief Get length in bytes of the buffer index buffer chain @@ -152,11 +162,12 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) always_inline u64 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) { - return vlib_physmem_offset_to_physical (&vm->physmem_main, - (((uword) buffer_index) << - CLIB_LOG2_CACHE_LINE_BYTES) + - STRUCT_OFFSET_OF (vlib_buffer_t, - data)); + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index); + vlib_buffer_pool_t *pool = vec_elt_at_index (bm->buffer_pools, + b->buffer_pool_index); + + return vlib_physmem_virtual_to_physical (vm, pool->physmem_region, b->data); } /** \brief Prefetch buffer metadata by buffer index @@ -205,13 +216,19 @@ typedef enum VLIB_BUFFER_KNOWN_ALLOCATED, } vlib_buffer_known_state_t; +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, + uword n_buffers, + vlib_buffer_known_state_t + expected_state); + always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + clib_spinlock_lock (&bm->buffer_known_hash_lockp); uword *p = hash_get (bm->buffer_known_hash, buffer_index); + clib_spinlock_unlock (&bm->buffer_known_hash_lockp); return p ? p[0] : VLIB_BUFFER_UNKNOWN; } @@ -221,8 +238,9 @@ vlib_buffer_set_known_state (vlib_main_t * vm, vlib_buffer_known_state_t state) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + clib_spinlock_lock (&bm->buffer_known_hash_lockp); hash_set (bm->buffer_known_hash, buffer_index, state); + clib_spinlock_unlock (&bm->buffer_known_hash_lockp); } /* Validates sanity of a single buffer. @@ -230,10 +248,31 @@ vlib_buffer_set_known_state (vlib_main_t * vm, u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, - unsigned socket_id); +always_inline u32 +vlib_buffer_round_size (u32 size) +{ + return round_pow2 (size, sizeof (vlib_buffer_t)); +} -/** \brief Allocate buffers into supplied array +always_inline u32 +vlib_buffer_get_free_list_index (vlib_buffer_t * b) +{ + return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK; +} + +always_inline void +vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) +{ + /* if there is an need for more free lists we should consider + storig data in the 2nd cacheline */ + ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1); + ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK); + + b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK; + b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK; +} + +/** \brief Allocate buffers from specific freelist into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @@ -242,22 +281,55 @@ clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, less than the number requested or zero */ always_inline u32 -vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +vlib_buffer_alloc_from_free_list (vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *fl; + u32 *src; + uword len; - ASSERT (bm->cb.vlib_buffer_alloc_cb); + ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); - return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers); -} + fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); -always_inline u32 -vlib_buffer_round_size (u32 size) -{ - return round_pow2 (size, sizeof (vlib_buffer_t)); + len = vec_len (fl->buffers); + + if (PREDICT_FALSE (len < n_buffers)) + { + bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); + len = vec_len (fl->buffers); + + /* even if fill free list didn't manage to refill free list + we should give what we have */ + n_buffers = clib_min (len, n_buffers); + + /* following code is intentionaly duplicated to allow compiler + to optimize fast path when n_buffers is constant value */ + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; + } + + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; } -/** \brief Allocate buffers from specific freelist into supplied array +/** \brief Allocate buffers into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @@ -266,16 +338,10 @@ vlib_buffer_round_size (u32 size) less than the number requested or zero */ always_inline u32 -vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) +vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb); - - return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers, - free_list_index); + return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); } /** \brief Free buffers @@ -377,7 +443,7 @@ vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b, vlib_buffer_main_t *bm = vm->buffer_main; u32 i; - *index = i = b->free_list_index; + *index = i = vlib_buffer_get_free_list_index (b); return pool_elt_at_index (bm->buffer_free_list_pool, i); } @@ -428,43 +494,6 @@ vlib_copy_buffers (u32 * dst, u32 * src, u32 n) } } -always_inline void * -vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error, - uword n_bytes, uword alignment) -{ - void *r = - vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment); - if (!r) - *error = - clib_error_return (0, "failed to allocate %wd bytes of I/O memory", - n_bytes); - else - *error = 0; - return r; -} - -/* By default allocate I/O memory with cache line alignment. */ -always_inline void * -vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes) -{ - return vlib_physmem_alloc_aligned (vm, error, n_bytes, - CLIB_CACHE_LINE_BYTES); -} - -always_inline void -vlib_physmem_free (vlib_main_t * vm, void *mem) -{ - return vm->os_physmem_free (mem); -} - -always_inline u64 -vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem) -{ - vlib_physmem_main_t *pm = &vm->physmem_main; - uword o = pointer_to_uword (mem) - pm->virtual.start; - return vlib_physmem_offset_to_physical (pm, o); -} - /* Append given data to end of buffer, possibly allocating new buffers. */ u32 vlib_buffer_add_data (vlib_main_t * vm, u32 free_list_index, @@ -565,7 +594,8 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, } n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - s->free_list_index); + vlib_buffer_get_free_list_index + (s)); if (PREDICT_FALSE (n_buffers == 0)) { buffers[0] = src_buffer; @@ -577,7 +607,8 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); d->current_data = s->current_data; d->current_length = head_end_offset; - d->free_list_index = s->free_list_index; + vlib_buffer_set_free_list_index (d, + vlib_buffer_get_free_list_index (s)); d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer + s->current_length - head_end_offset; @@ -611,7 +642,8 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT (head->free_list_index == tail->free_list_index); + ASSERT (vlib_buffer_get_free_list_index (head) == + vlib_buffer_get_free_list_index (tail)); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -787,7 +819,7 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, CLIB_CACHE_LINE_BYTES * 2); /* Make sure buffer template is sane. */ - ASSERT (fl->index == fl->buffer_init_template.free_list_index); + ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); clib_memcpy (STRUCT_MARK_PTR (dst, template_start), STRUCT_MARK_PTR (src, template_start), @@ -802,9 +834,13 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, _(current_data); _(current_length); _(flags); - _(free_list_index); #undef _ - ASSERT (dst->total_length_not_including_first_buffer == 0); + /* ASSERT (dst->total_length_not_including_first_buffer == 0); */ + /* total_length_not_including_first_buffer is not in the template anymore + * so it may actually not zeroed for some buffers. One option is to + * uncomment the line lower (comes at a cost), the other, is to just not + * care */ + /* dst->total_length_not_including_first_buffer = 0; */ ASSERT (dst->n_add_refs == 0); } @@ -818,6 +854,19 @@ vlib_buffer_add_to_free_list (vlib_main_t * vm, if (PREDICT_TRUE (do_init)) vlib_buffer_init_for_free_list (b, f); vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES); + + if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE) + { + vlib_buffer_free_list_t *mf; + mf = vlib_buffer_get_free_list (vlib_mains[0], f->index); + clib_spinlock_lock (&mf->global_buffers_lock); + /* keep last stored buffers, as they are more likely hot in the cache */ + vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_delete (f->buffers, VLIB_FRAME_SIZE, 0); + f->n_alloc -= VLIB_FRAME_SIZE; + clib_spinlock_unlock (&mf->global_buffers_lock); + } } always_inline void @@ -828,7 +877,7 @@ vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, vlib_buffer_t *src = &fl->buffer_init_template; /* Make sure buffer template is sane. */ - ASSERT (fl->index == fl->buffer_init_template.free_list_index); + ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); clib_memcpy (STRUCT_MARK_PTR (dst0, template_start), STRUCT_MARK_PTR (src, template_start), @@ -849,7 +898,6 @@ vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, _(current_data); _(current_length); _(flags); - _(free_list_index); #undef _ ASSERT (dst0->total_length_not_including_first_buffer == 0);