X-Git-Url: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=21e91026b3e5345d353e6720b0f940b6643d7e6c;hp=4f5eb09da91656976172a2e4a8bd3996024c77eb;hb=8feeaff56;hpb=c47ed032c6d036a9f942fc9ced48874fad55b48c diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 4f5eb09da91..21e91026b3e 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -43,7 +43,36 @@ * Allocate/free network buffers. */ +#include #include +#include +#include + +#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 +#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 + +#ifdef CLIB_HAVE_VEC128 +/* Assumptions by vlib_buffer_free_inline: */ +STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16); +STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16); +STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16); +#endif + +/* Make sure that buffer template size is not accidentally changed */ +STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); + +u16 __vlib_buffer_external_hdr_size = 0; + +static void +buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, + u32 index); + +static void +buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e, + u32 index); + +static void +buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index); uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, @@ -66,11 +95,17 @@ u8 * format_vlib_buffer (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); + u8 *a = 0; - s = format (s, "current data %d, length %d, free-list %d, clone-count %u", - b->current_data, b->current_length, b->free_list_index, - b->n_add_refs); +#define _(bit, name, v) \ + if (v && (b->flags & VLIB_BUFFER_##name)) \ + a = format (a, "%s ", v); + foreach_vlib_buffer_flag +#undef _ + s = format (s, "current data %d, length %d, buffer-pool %d, " + "ref-count %u", b->current_data, b->current_length, + b->buffer_pool_index, b->ref_count); if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) s = format (s, ", totlen-nifb %d", @@ -79,6 +114,10 @@ format_vlib_buffer (u8 * s, va_list * args) if (b->flags & VLIB_BUFFER_IS_TRACED) s = format (s, ", trace 0x%x", b->trace_index); + if (a) + s = format (s, "\n%U%v", format_white_space, indent, a); + vec_free (a); + while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { vlib_main_t *vm = vlib_get_main (); @@ -86,9 +125,9 @@ format_vlib_buffer (u8 * s, va_list * args) b = vlib_get_buffer (vm, next_buffer); s = - format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u", + format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u", format_white_space, indent, next_buffer, b->current_length, - b->n_add_refs); + b->ref_count); } return s; @@ -156,21 +195,19 @@ vlib_validate_buffer_helper (vlib_main_t * vm, u32 bi, uword follow_buffer_next, uword ** unique_hash) { - vlib_buffer_t *b = vlib_get_buffer (vm, bi); vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *fl; - - if (pool_is_free_index (bm->buffer_free_list_pool, b->free_list_index)) - return format (0, "unknown free list 0x%x", b->free_list_index); + vlib_buffer_t *b = vlib_get_buffer (vm, bi); - fl = pool_elt_at_index (bm->buffer_free_list_pool, b->free_list_index); + if (vec_len (bm->buffer_pools) <= b->buffer_pool_index) + return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index); if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); - if (b->current_data + b->current_length > fl->n_data_bytes) - return format (0, "%d-%d beyond end of buffer %d", - b->current_data, b->current_length, fl->n_data_bytes); + if (b->current_data + b->current_length > + vlib_buffer_get_default_data_size (vm)) + return format (0, "%d-%d beyond end of buffer %d", b->current_data, + b->current_length, vlib_buffer_get_default_data_size (vm)); if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) { @@ -261,28 +298,44 @@ done: return result; } -vlib_main_t **vlib_mains; +/* + * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1 + * and vlib_mains[0] = &vlib_global_main from the beginning of time. + * + * The only place which should ever expand vlib_mains is start_workers() + * in threads.c. It knows about the bootstrap vector. + */ +/* *INDENT-OFF* */ +static struct +{ + vec_header_t h; + vlib_main_t *vm; +} __attribute__ ((packed)) __bootstrap_vlib_main_vector + __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) = +{ + .h.len = 1, + .vm = &vlib_global_main, +}; +/* *INDENT-ON* */ + +vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; -/* When dubugging validate that given buffers are either known allocated + +/* When debugging validate that given buffers are either known allocated or known free. */ -static void +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, uword n_buffers, vlib_buffer_known_state_t expected_state) { + vlib_buffer_main_t *bm = vm->buffer_main; u32 *b; uword i, bi, is_free; if (CLIB_DEBUG == 0) return; - ASSERT (os_get_cpu_number () == 0); - - /* smp disaster check */ - if (vlib_mains) - ASSERT (vm == vlib_mains[0]); - is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; b = buffers; for (i = 0; i < n_buffers; i++) @@ -292,448 +345,21 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, bi = b[0]; b += 1; known = vlib_buffer_is_known (vm, bi); - if (known != expected_state) - { - ASSERT (0); - vlib_panic_with_msg - (vm, "%s %U buffer 0x%x", - is_free ? "freeing" : "allocating", - format_vlib_buffer_known_state, known, bi); - } - - vlib_buffer_set_known_state - (vm, bi, - is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED); - } -} - -void -vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst, - vlib_buffer_free_list_t * src) -{ - uword l; - u32 *d; - - l = vec_len (src->buffers); - if (l > 0) - { - vec_add2_aligned (dst->buffers, d, l, CLIB_CACHE_LINE_BYTES); - clib_memcpy (d, src->buffers, l * sizeof (d[0])); - vec_free (src->buffers); - } -} - -/* Add buffer free list. */ -static u32 -vlib_buffer_create_free_list_helper (vlib_main_t * vm, - u32 n_data_bytes, - u32 is_public, u32 is_default, u8 * name) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - int i; - - ASSERT (os_get_cpu_number () == 0); - - if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) - { - u32 default_free_free_list_index; - - /* *INDENT-OFF* */ - default_free_free_list_index = - vlib_buffer_create_free_list_helper - (vm, - /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, - /* is_public */ 1, - /* is_default */ 1, - (u8 *) "default"); - /* *INDENT-ON* */ - ASSERT (default_free_free_list_index == - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public) - return default_free_free_list_index; - } - - pool_get_aligned (bm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES); - - memset (f, 0, sizeof (f[0])); - f->index = f - bm->buffer_free_list_pool; - f->n_data_bytes = vlib_buffer_round_size (n_data_bytes); - f->min_n_buffers_each_physmem_alloc = 16; - f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name); - - /* Setup free buffer template. */ - f->buffer_init_template.free_list_index = f->index; - f->buffer_init_template.n_add_refs = 0; - - if (is_public) - { - uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes); - if (!p) - hash_set (bm->free_list_by_size, f->n_data_bytes, f->index); - } - - for (i = 1; i < vec_len (vlib_mains); i++) - { - vlib_buffer_main_t *wbm = vlib_mains[i]->buffer_main; - vlib_buffer_free_list_t *wf; - pool_get_aligned (wbm->buffer_free_list_pool, - wf, CLIB_CACHE_LINE_BYTES); - ASSERT (f - bm->buffer_free_list_pool == - wf - wbm->buffer_free_list_pool); - wf[0] = f[0]; - wf->buffers = 0; - wf->n_alloc = 0; - } - - return f->index; -} - -u32 -vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...) -{ - va_list va; - u8 *name; - - va_start (va, fmt); - name = va_format (0, fmt, &va); - va_end (va); - return vlib_buffer_create_free_list_helper (vm, n_data_bytes, - /* is_public */ 0, - /* is_default */ 0, - name); -} - -u32 -vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...) -{ - u32 i = vlib_buffer_get_free_list_with_size (vm, n_data_bytes); - - if (i == ~0) - { - va_list va; - u8 *name; - - va_start (va, fmt); - name = va_format (0, fmt, &va); - va_end (va); - - i = vlib_buffer_create_free_list_helper (vm, n_data_bytes, - /* is_public */ 1, - /* is_default */ 0, - name); - } - - return i; -} - -static void -del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) -{ - u32 i; - - for (i = 0; i < vec_len (f->buffer_memory_allocated); i++) - vm->os_physmem_free (f->buffer_memory_allocated[i]); - vec_free (f->name); - vec_free (f->buffer_memory_allocated); - vec_free (f->buffers); -} + if (known == VLIB_BUFFER_UNKNOWN && + expected_state == VLIB_BUFFER_KNOWN_FREE) + known = VLIB_BUFFER_KNOWN_FREE; -/* Add buffer free list. */ -void -vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - u32 merge_index; - int i; - - ASSERT (os_get_cpu_number () == 0); - - f = vlib_buffer_get_free_list (vm, free_list_index); - - ASSERT (vec_len (f->buffers) == f->n_alloc); - merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); - if (merge_index != ~0 && merge_index != free_list_index) - { - vlib_buffer_merge_free_lists (pool_elt_at_index - (bm->buffer_free_list_pool, merge_index), - f); - } - - del_free_list (vm, f); - - /* Poison it. */ - memset (f, 0xab, sizeof (f[0])); - - pool_put (bm->buffer_free_list_pool, f); - - for (i = 1; i < vec_len (vlib_mains); i++) - { - bm = vlib_mains[i]->buffer_main; - f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);; - memset (f, 0xab, sizeof (f[0])); - pool_put (bm->buffer_free_list_pool, f); - } -} - -/* Make sure free list has at least given number of free buffers. */ -static uword -fill_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, uword min_free_buffers) -{ - vlib_buffer_t *buffers, *b; - int n, n_bytes, i; - u32 *bi; - u32 n_remaining, n_alloc, n_this_chunk; - - /* Already have enough free buffers on free list? */ - n = min_free_buffers - vec_len (fl->buffers); - if (n <= 0) - return min_free_buffers; - - /* Always allocate round number of buffers. */ - n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32)); - - /* Always allocate new buffers in reasonably large sized chunks. */ - n = clib_max (n, fl->min_n_buffers_each_physmem_alloc); - - n_remaining = n; - n_alloc = 0; - while (n_remaining > 0) - { - n_this_chunk = clib_min (n_remaining, 16); - - n_bytes = n_this_chunk * (sizeof (b[0]) + fl->n_data_bytes); - - /* drb: removed power-of-2 ASSERT */ - buffers = vm->os_physmem_alloc_aligned (&vm->physmem_main, - n_bytes, - sizeof (vlib_buffer_t)); - if (!buffers) - return n_alloc; - - /* Record chunk as being allocated so we can free it later. */ - vec_add1 (fl->buffer_memory_allocated, buffers); - - fl->n_alloc += n_this_chunk; - n_alloc += n_this_chunk; - n_remaining -= n_this_chunk; - - b = buffers; - vec_add2_aligned (fl->buffers, bi, n_this_chunk, CLIB_CACHE_LINE_BYTES); - for (i = 0; i < n_this_chunk; i++) - { - bi[i] = vlib_get_buffer_index (vm, b); - - if (CLIB_DEBUG > 0) - vlib_buffer_set_known_state (vm, bi[i], VLIB_BUFFER_KNOWN_FREE); - b = vlib_buffer_next_contiguous (b, fl->n_data_bytes); - } - - memset (buffers, 0, n_bytes); - - /* Initialize all new buffers. */ - b = buffers; - for (i = 0; i < n_this_chunk; i++) - { - vlib_buffer_init_for_free_list (b, fl); - b = vlib_buffer_next_contiguous (b, fl->n_data_bytes); - } - - if (fl->buffer_init_function) - fl->buffer_init_function (vm, fl, bi, n_this_chunk); - } - return n_alloc; -} - -static u32 -alloc_from_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * free_list, - u32 * alloc_buffers, u32 n_alloc_buffers) -{ - u32 *dst, *src; - uword len; - uword n_filled; - - dst = alloc_buffers; - - n_filled = fill_free_list (vm, free_list, n_alloc_buffers); - if (n_filled == 0) - return 0; - - len = vec_len (free_list->buffers); - ASSERT (len >= n_alloc_buffers); - - src = free_list->buffers + len - n_alloc_buffers; - clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32)); - - _vec_len (free_list->buffers) -= n_alloc_buffers; - - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, alloc_buffers, - n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE); - - return n_alloc_buffers; -} - - -/* Allocate a given number of buffers into given array. - Returns number actually allocated which will be either zero or - number requested. */ -static u32 -vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - return alloc_from_free_list - (vm, - pool_elt_at_index (bm->buffer_free_list_pool, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), - buffers, n_buffers); -} - -static u32 -vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); - return alloc_from_free_list (vm, f, buffers, n_buffers); -} - -void * -vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - void *rv = bm->buffer_free_callback; - - bm->buffer_free_callback = fp; - return rv; -} - -static_always_inline void -vlib_buffer_free_inline (vlib_main_t * vm, - u32 * buffers, u32 n_buffers, u32 follow_buffer_next) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *fl; - u32 fi; - int i; - u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, - u32 follow_buffer_next); - - cb = bm->buffer_free_callback; - - if (PREDICT_FALSE (cb != 0)) - n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); - - if (!n_buffers) - return; - - for (i = 0; i < n_buffers; i++) - { - vlib_buffer_t *b; - u32 bi = buffers[i]; - - b = vlib_get_buffer (vm, bi); - - fl = vlib_buffer_get_buffer_free_list (vm, b, &fi); - - /* The only current use of this callback: multicast recycle */ - if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) - { - int j; - - vlib_buffer_add_to_free_list - (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0); - - for (j = 0; j < vec_len (bm->announce_list); j++) - { - if (fl == bm->announce_list[j]) - goto already_announced; - } - vec_add1 (bm->announce_list, fl); - already_announced: - ; - } - else - { - if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) - { - u32 flags, next; - - do - { - vlib_buffer_t *nb = vlib_get_buffer (vm, bi); - flags = nb->flags; - next = nb->next_buffer; - if (nb->n_add_refs) - nb->n_add_refs--; - else - { - vlib_buffer_validate_alloc_free (vm, &bi, 1, - VLIB_BUFFER_KNOWN_ALLOCATED); - vlib_buffer_add_to_free_list (vm, fl, bi, 1); - } - bi = next; - } - while (follow_buffer_next - && (flags & VLIB_BUFFER_NEXT_PRESENT)); - - } - } - } - if (vec_len (bm->announce_list)) - { - vlib_buffer_free_list_t *fl; - for (i = 0; i < vec_len (bm->announce_list); i++) + if (known != expected_state) { - fl = bm->announce_list[i]; - fl->buffers_added_to_freelist_function (vm, fl); + clib_panic ("%s %U buffer 0x%x", is_free ? "freeing" : "allocating", + format_vlib_buffer_known_state, known, bi); } - _vec_len (bm->announce_list) = 0; - } -} - -static void -vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 1); -} -static void -vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers, - u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 0); -} - -/* Copy template packet data into buffers as they are allocated. */ -static void __attribute__ ((unused)) -vlib_packet_template_buffer_init (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, - u32 * buffers, u32 n_buffers) -{ - vlib_packet_template_t *t = - uword_to_pointer (fl->buffer_init_function_opaque, - vlib_packet_template_t *); - uword i; - - for (i = 0; i < n_buffers; i++) - { - vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); - ASSERT (b->current_length == vec_len (t->packet_data)); - clib_memcpy (vlib_buffer_get_current (b), t->packet_data, - b->current_length); + clib_spinlock_lock (&bm->buffer_known_hash_lockp); + hash_set (bm->buffer_known_hash, bi, is_free ? VLIB_BUFFER_KNOWN_FREE : + VLIB_BUFFER_KNOWN_ALLOCATED); + clib_spinlock_unlock (&bm->buffer_known_hash_lockp); } } @@ -742,48 +368,20 @@ vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, void *packet_data, uword n_packet_data_bytes, - uword min_n_buffers_each_physmem_alloc, - char *fmt, ...) + uword min_n_buffers_each_alloc, char *fmt, ...) { - vlib_buffer_main_t *bm = vm->buffer_main; va_list va; - __attribute__ ((unused)) u8 *name; - vlib_buffer_free_list_t *fl; va_start (va, fmt); - name = va_format (0, fmt, &va); + t->name = va_format (0, fmt, &va); va_end (va); - if (bm->cb.vlib_packet_template_init_cb) - bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data, - n_packet_data_bytes, - min_n_buffers_each_physmem_alloc, - name); - vlib_worker_thread_barrier_sync (vm); - memset (t, 0, sizeof (t[0])); + clib_memset (t, 0, sizeof (t[0])); vec_add (t->packet_data, packet_data, n_packet_data_bytes); - t->min_n_buffers_each_physmem_alloc = min_n_buffers_each_physmem_alloc; - - t->free_list_index = vlib_buffer_create_free_list_helper - (vm, n_packet_data_bytes, - /* is_public */ 1, - /* is_default */ 0, - name); - - ASSERT (t->free_list_index != 0); - fl = vlib_buffer_get_free_list (vm, t->free_list_index); - fl->min_n_buffers_each_physmem_alloc = t->min_n_buffers_each_physmem_alloc; - - fl->buffer_init_function = vlib_packet_template_buffer_init; - fl->buffer_init_function_opaque = pointer_to_uword (t); - - fl->buffer_init_template.current_data = 0; - fl->buffer_init_template.current_length = n_packet_data_bytes; - fl->buffer_init_template.flags = 0; - fl->buffer_init_template.n_add_refs = 0; + t->min_n_buffers_each_alloc = min_n_buffers_each_alloc; vlib_worker_thread_barrier_release (vm); } @@ -800,48 +398,29 @@ vlib_packet_template_get_packet (vlib_main_t * vm, *bi_result = bi; b = vlib_get_buffer (vm, bi); - clib_memcpy (vlib_buffer_get_current (b), - t->packet_data, vec_len (t->packet_data)); + clib_memcpy_fast (vlib_buffer_get_current (b), + t->packet_data, vec_len (t->packet_data)); b->current_length = vec_len (t->packet_data); return b->data; } -void -vlib_packet_template_get_packet_helper (vlib_main_t * vm, - vlib_packet_template_t * t) -{ - word n = t->min_n_buffers_each_physmem_alloc; - word l = vec_len (t->packet_data); - word n_alloc; - - ASSERT (l > 0); - ASSERT (vec_len (t->free_buffers) == 0); - - vec_validate (t->free_buffers, n - 1); - n_alloc = vlib_buffer_alloc_from_free_list (vm, t->free_buffers, - n, t->free_list_index); - _vec_len (t->free_buffers) = n_alloc; -} - /* Append given data to end of buffer, possibly allocating new buffers. */ -u32 -vlib_buffer_add_data (vlib_main_t * vm, - u32 free_list_index, - u32 buffer_index, void *data, u32 n_data_bytes) +int +vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, + u32 n_data_bytes) { u32 n_buffer_bytes, n_left, n_left_this_buffer, bi; vlib_buffer_t *b; void *d; - bi = buffer_index; - if (bi == 0 - && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index)) + bi = *buffer_index; + if (bi == ~0 && 1 != vlib_buffer_alloc (vm, &bi, 1)) goto out_of_buffers; d = data; n_left = n_data_bytes; - n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index); + n_buffer_bytes = vlib_buffer_get_default_data_size (vm); b = vlib_get_buffer (vm, bi); b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -858,16 +437,15 @@ vlib_buffer_add_data (vlib_main_t * vm, n_left_this_buffer = n_buffer_bytes - (b->current_data + b->current_length); n = clib_min (n_left_this_buffer, n_left); - clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n); + clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d, + n); b->current_length += n; n_left -= n; if (n_left == 0) break; d += n; - if (1 != - vlib_buffer_alloc_from_free_list (vm, &b->next_buffer, 1, - free_list_index)) + if (1 != vlib_buffer_alloc (vm, &b->next_buffer, 1)) goto out_of_buffers; b->flags |= VLIB_BUFFER_NEXT_PRESENT; @@ -875,23 +453,22 @@ vlib_buffer_add_data (vlib_main_t * vm, b = vlib_get_buffer (vm, b->next_buffer); } - return bi; + *buffer_index = bi; + return 0; out_of_buffers: - clib_error ("out of buffers"); - return bi; + clib_warning ("out of buffers"); + return 1; } u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - u32 free_list_index, vlib_buffer_t * first, - vlib_buffer_t ** last, - void *data, u16 data_len) + vlib_buffer_t ** last, void *data, + u16 data_len) { vlib_buffer_t *l = *last; - u32 n_buffer_bytes = - vlib_buffer_free_list_buffer_size (vm, free_list_index); + u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); u16 copied = 0; ASSERT (n_buffer_bytes >= l->current_length + l->current_data); while (data_len) @@ -899,17 +476,16 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, u16 max = n_buffer_bytes - l->current_length - l->current_data; if (max == 0) { - if (1 != - vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, - free_list_index)) + if (1 != vlib_buffer_alloc_from_pool (vm, &l->next_buffer, 1, + first->buffer_pool_index)) return copied; - *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer); + *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer); max = n_buffer_bytes - l->current_length - l->current_data; } u16 len = (data_len > max) ? max : data_len; - clib_memcpy (vlib_buffer_get_current (l) + l->current_length, - data + copied, len); + clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length, + data + copied, len); vlib_buffer_chain_increase_length (first, l, len); data_len -= len; copied += len; @@ -917,28 +493,115 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, return copied; } - -static u8 * -format_vlib_buffer_free_list (u8 * s, va_list * va) +u8 +vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size, + u32 physmem_map_index) { - vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *); - u32 threadnum = va_arg (*va, u32); - uword bytes_alloc, bytes_free, n_free, size; + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; + vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); + uword start = pointer_to_uword (m->base); + uword size = (uword) m->n_pages << m->log2_page_size; + uword i, j; + u32 alloc_size, n_alloc_per_page; - if (!f) - return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s", - "Thread", "Name", "Index", "Size", "Alloc", "Free", - "#Alloc", "#Free"); + if (vec_len (bm->buffer_pools) >= 255) + return ~0; - size = sizeof (vlib_buffer_t) + f->n_data_bytes; - n_free = vec_len (f->buffers); - bytes_alloc = size * f->n_alloc; - bytes_free = size * n_free; + vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_LOG2_CACHE_LINE_BYTES); - s = format (s, "%7d%30s%12d%12d%=12U%=12U%=12d%=12d", threadnum, - f->name, f->index, f->n_data_bytes, - format_memory_size, bytes_alloc, - format_memory_size, bytes_free, f->n_alloc, n_free); + if (bm->buffer_mem_size == 0) + { + bm->buffer_mem_start = start; + bm->buffer_mem_size = size; + } + else if (start < bm->buffer_mem_start) + { + bm->buffer_mem_size += bm->buffer_mem_start - start; + bm->buffer_mem_start = start; + if (size > bm->buffer_mem_size) + bm->buffer_mem_size = size; + } + else if (start > bm->buffer_mem_start) + { + uword new_size = start - bm->buffer_mem_start + size; + if (new_size > bm->buffer_mem_size) + bm->buffer_mem_size = new_size; + } + + if ((u64) bm->buffer_mem_size > + ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES))) + { + clib_panic ("buffer memory size out of range!"); + } + + bp->start = start; + bp->size = size; + bp->index = bp - bm->buffer_pools; + bp->buffer_template.buffer_pool_index = bp->index; + bp->buffer_template.ref_count = 1; + bp->physmem_map_index = physmem_map_index; + bp->name = format (0, "%s%c", name, 0); + bp->data_size = data_size; + bp->numa_node = m->numa_node; + + vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + CLIB_CACHE_LINE_BYTES); + + alloc_size = data_size + sizeof (vlib_buffer_t) + bm->ext_hdr_size; + n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size; + + /* preallocate buffer indices memory */ + vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (bp->buffers); + + clib_spinlock_init (&bp->lock); + + for (j = 0; j < m->n_pages; j++) + for (i = 0; i < n_alloc_per_page; i++) + { + u8 *p; + u32 bi; + + p = m->base + (j << m->log2_page_size) + i * alloc_size; + p += bm->ext_hdr_size; + + vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template); + + bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p); + + vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES); + vlib_get_buffer (vm, bi); + } + + bp->n_buffers = vec_len (bp->buffers); + return bp->index; +} + +static u8 * +format_vlib_buffer_pool (u8 * s, va_list * va) +{ + vlib_main_t *vm = va_arg (*va, vlib_main_t *); + vlib_buffer_pool_t *bp = va_arg (*va, vlib_buffer_pool_t *); + vlib_buffer_pool_thread_t *bpt; + u32 cached = 0; + + if (!bp) + return format (s, "%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s", + "Pool Name", "Index", "NUMA", "Size", "Data Size", + "Total", "Avail", "Cached", "Used"); + + /* *INDENT-OFF* */ + vec_foreach (bpt, bp->threads) + cached += vec_len (bpt->cached_buffers); + /* *INDENT-ON* */ + + s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", + bp->name, bp->index, bp->numa_node, bp->data_size + + sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size, + bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached, + bp->n_buffers - vec_len (bp->buffers) - cached); return s; } @@ -947,27 +610,15 @@ static clib_error_t * show_buffers (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - vlib_buffer_main_t *bm; - vlib_buffer_free_list_t *f; - vlib_main_t *curr_vm; - u32 vm_index = 0; - - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0, 0); + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; - do - { - curr_vm = vec_len (vlib_mains) ? vlib_mains[vm_index] : vm; - bm = curr_vm->buffer_main; + vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, 0); - /* *INDENT-OFF* */ - pool_foreach (f, bm->buffer_free_list_pool, ({ - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index); - })); - /* *INDENT-ON* */ - - vm_index++; - } - while (vm_index < vec_len (vlib_mains)); + /* *INDENT-OFF* */ + vec_foreach (bp, bm->buffer_pools) + vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, bp); + /* *INDENT-ON* */ return 0; } @@ -980,38 +631,283 @@ VLIB_CLI_COMMAND (show_buffers_command, static) = { }; /* *INDENT-ON* */ -void -vlib_buffer_cb_init (struct vlib_main_t *vm) +clib_error_t * +vlib_buffer_worker_init (vlib_main_t * vm) { vlib_buffer_main_t *bm = vm->buffer_main; - bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal; - bm->cb.vlib_buffer_alloc_from_free_list_cb = - &vlib_buffer_alloc_from_free_list_internal; - bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; - bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; - bm->cb.vlib_buffer_delete_free_list_cb = - &vlib_buffer_delete_free_list_internal; - bm->extern_buffer_mgmt = 0; + vlib_buffer_pool_t *bp; + + /* *INDENT-OFF* */ + vec_foreach (bp, bm->buffer_pools) + { + clib_spinlock_lock (&bp->lock); + vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + CLIB_CACHE_LINE_BYTES); + clib_spinlock_unlock (&bp->lock); + } + /* *INDENT-ON* */ + + return 0; } -int -vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb) +VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init); + +static clib_error_t * +vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node, + u8 * index) { vlib_buffer_main_t *bm = vm->buffer_main; - if (bm->extern_buffer_mgmt) - return -1; - -#define _(x) bm->cb.x = cb->x - _(vlib_buffer_alloc_cb); - _(vlib_buffer_alloc_from_free_list_cb); - _(vlib_buffer_free_cb); - _(vlib_buffer_free_no_next_cb); - _(vlib_buffer_delete_free_list_cb); -#undef _ - bm->extern_buffer_mgmt = 1; + clib_error_t *error; + u32 physmem_map_index; + uword n_pages, pagesize; + u32 buffers_per_numa; + u32 buffer_size = CLIB_CACHE_LINE_ROUND (bm->ext_hdr_size + + sizeof (vlib_buffer_t) + + vlib_buffer_get_default_data_size + (vm)); + u8 *name; + + pagesize = clib_mem_get_default_hugepage_size (); + name = format (0, "buffers-numa-%d%c", numa_node, 0); + + buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; + +retry: + n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1; + error = vlib_physmem_shared_map_create (vm, (char *) name, + n_pages * pagesize, + min_log2 (pagesize), numa_node, + &physmem_map_index); + + if (error && pagesize != clib_mem_get_page_size ()) + { + vlib_log_warn (bm->log_default, "%U", format_clib_error, error); + clib_error_free (error); + vlib_log_warn (bm->log_default, "falling back to non-hugepage " + "backed buffer pool"); + pagesize = clib_mem_get_page_size (); + buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV; + goto retry; + } + + if (error) + return error; + + vec_reset_length (name); + name = format (name, "default-numa-%d%c", numa_node, 0); + + *index = vlib_buffer_pool_create (vm, (char *) name, + vlib_buffer_get_default_data_size (vm), + physmem_map_index); + + if (*index == (u8) ~ 0) + return clib_error_return (0, "maximum number of buffer pools reached"); + return 0; } +void +vlib_buffer_main_alloc (vlib_main_t * vm) +{ + vlib_buffer_main_t *bm; + + if (vm->buffer_main) + return; + + vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0])); + clib_memset (vm->buffer_main, 0, sizeof (bm[0])); + bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; +} + +static u32 +buffer_get_cached (vlib_buffer_pool_t * bp) +{ + u32 cached = 0; + vlib_buffer_pool_thread_t *bpt; + + clib_spinlock_lock (&bp->lock); + + /* *INDENT-OFF* */ + vec_foreach (bpt, bp->threads) + cached += vec_len (bpt->cached_buffers); + /* *INDENT-ON* */ + + clib_spinlock_unlock (&bp->lock); + + return cached; +} + +static vlib_buffer_pool_t * +buffer_get_by_index (vlib_buffer_main_t * bm, u32 index) +{ + vlib_buffer_pool_t *bp; + if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index) + return 0; + bp = vec_elt_at_index (bm->buffer_pools, index); + + return bp; +} + +static void +buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = bp->n_buffers - vec_len (bp->buffers) - buffer_get_cached (bp); +} + +static void +buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e, + u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = vec_len (bp->buffers); +} + +static void +buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = buffer_get_cached (bp); +} + +clib_error_t * +vlib_buffer_main_init (struct vlib_main_t * vm) +{ + vlib_buffer_main_t *bm; + clib_error_t *err; + clib_bitmap_t *bmp = 0, *bmp_has_memory = 0; + u32 numa_node; + vlib_buffer_pool_t *bp; + u8 *name = 0, first_valid_buffer_pool_index = ~0; + + vlib_buffer_main_alloc (vm); + + bm = vm->buffer_main; + bm->log_default = vlib_log_register_class ("buffer", 0); + bm->ext_hdr_size = __vlib_buffer_external_hdr_size; + + clib_spinlock_init (&bm->buffer_known_hash_lockp); + + if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U", + unformat_bitmap_list, &bmp))) + clib_error_free (err); + + if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U", + unformat_bitmap_list, &bmp_has_memory))) + clib_error_free (err); + + if (bmp && bmp_has_memory) + bmp = clib_bitmap_and (bmp, bmp_has_memory); + + /* no info from sysfs, assuming that only numa 0 exists */ + if (bmp == 0) + bmp = clib_bitmap_set (bmp, 0, 1); + + if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES) + clib_panic ("system have more than %u NUMA nodes", + VLIB_BUFFER_MAX_NUMA_NODES); + + /* *INDENT-OFF* */ + clib_bitmap_foreach (numa_node, bmp, + { + u8 *index = bm->default_buffer_pool_index_for_numa + numa_node; + index[0] = ~0; + if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index))) + { + clib_error_report (err); + clib_error_free (err); + continue; + } + + if (first_valid_buffer_pool_index == 0xff) + first_valid_buffer_pool_index = index[0]; + }); + /* *INDENT-ON* */ + + if (first_valid_buffer_pool_index == (u8) ~ 0) + { + err = clib_error_return (0, "failed to allocate buffer pool(s)"); + goto done; + } + + /* *INDENT-OFF* */ + clib_bitmap_foreach (numa_node, bmp, + { + if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0) + bm->default_buffer_pool_index_for_numa[numa_node] = + first_valid_buffer_pool_index; + }); + /* *INDENT-ON* */ + + vec_foreach (bp, bm->buffer_pools) + { + if (bp->n_buffers == 0) + continue; + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/cached%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_cached_fn, + bp - bm->buffer_pools); + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/used%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_used_fn, + bp - bm->buffer_pools); + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/available%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_available_fn, + bp - bm->buffer_pools); + } + +done: + vec_free (bmp); + vec_free (bmp_has_memory); + vec_free (name); + return err; +} + +static clib_error_t * +vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) +{ + vlib_buffer_main_t *bm; + + vlib_buffer_main_alloc (vm); + + bm = vm->buffer_main; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa)) + ; + else if (unformat (input, "default data-size %u", + &bm->default_data_size)) + ; + else + return unformat_parse_error (input); + } + + unformat_free (input); + return 0; +} + +VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers"); + + /** @endcond */ /* * fd.io coding-style-patch-verification: ON