X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=ae88b4e4d44ab0a427a7c2edd715c0822fe2229b;hb=ae8d332699685a8527fe42eb9be0bca4d639d5c3;hp=701ddde2227887bdd3f700c9a244f75065500eca;hpb=24dcbe45209dfdbcf2f1851aa443e696fac050f2;p=vpp.git diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 701ddde2227..ae88b4e4d44 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -46,6 +46,7 @@ #include #include #include +#include #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 @@ -62,6 +63,17 @@ STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); u16 __vlib_buffer_external_hdr_size = 0; +static void +buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, + u32 index); + +static void +buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e, + u32 index); + +static void +buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index); + uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) @@ -80,7 +92,7 @@ vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, } u8 * -format_vlib_buffer (u8 * s, va_list * args) +format_vlib_buffer_no_chain (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); u32 indent = format_get_indent (s); @@ -100,15 +112,26 @@ format_vlib_buffer (u8 * s, va_list * args) b->total_length_not_including_first_buffer); if (b->flags & VLIB_BUFFER_IS_TRACED) - s = format (s, ", trace 0x%x", b->trace_index); + s = format (s, ", trace handle 0x%x", b->trace_handle); if (a) s = format (s, "\n%U%v", format_white_space, indent, a); vec_free (a); + return s; +} + +u8 * +format_vlib_buffer (u8 * s, va_list * args) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); + u32 indent = format_get_indent (s); + + s = format (s, "%U", format_vlib_buffer_no_chain, b); + while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { - vlib_main_t *vm = vlib_get_main (); u32 next_buffer = b->next_buffer; b = vlib_get_buffer (vm, next_buffer); @@ -192,9 +215,10 @@ vlib_validate_buffer_helper (vlib_main_t * vm, if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); - if (b->current_data + b->current_length > VLIB_BUFFER_DATA_SIZE) + if (b->current_data + b->current_length > + vlib_buffer_get_default_data_size (vm)) return format (0, "%d-%d beyond end of buffer %d", b->current_data, - b->current_length, VLIB_BUFFER_DATA_SIZE); + b->current_length, vlib_buffer_get_default_data_size (vm)); if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) { @@ -285,30 +309,7 @@ done: return result; } -/* - * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1 - * and vlib_mains[0] = &vlib_global_main from the beginning of time. - * - * The only place which should ever expand vlib_mains is start_workers() - * in threads.c. It knows about the bootstrap vector. - */ -/* *INDENT-OFF* */ -static struct -{ - vec_header_t h; - vlib_main_t *vm; -} __attribute__ ((packed)) __bootstrap_vlib_main_vector - __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) = -{ - .h.len = 1, - .vm = &vlib_global_main, -}; -/* *INDENT-ON* */ - -vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; - - -/* When dubugging validate that given buffers are either known allocated +/* When debugging validate that given buffers are either known allocated or known free. */ void vlib_buffer_validate_alloc_free (vlib_main_t * vm, @@ -339,10 +340,8 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, if (known != expected_state) { - vlib_panic_with_msg - (vm, "%s %U buffer 0x%x", - is_free ? "freeing" : "allocating", - format_vlib_buffer_known_state, known, bi); + clib_panic ("%s %U buffer 0x%x", is_free ? "freeing" : "allocating", + format_vlib_buffer_known_state, known, bi); } clib_spinlock_lock (&bm->buffer_known_hash_lockp); @@ -409,7 +408,7 @@ vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, d = data; n_left = n_data_bytes; - n_buffer_bytes = VLIB_BUFFER_DATA_SIZE; + n_buffer_bytes = vlib_buffer_get_default_data_size (vm); b = vlib_get_buffer (vm, bi); b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -457,7 +456,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, u16 data_len) { vlib_buffer_t *l = *last; - u32 n_buffer_bytes = VLIB_BUFFER_DATA_SIZE; + u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); u16 copied = 0; ASSERT (n_buffer_bytes >= l->current_length + l->current_data); while (data_len) @@ -482,26 +481,35 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, return copied; } -clib_error_t * -vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name, - u32 data_size, u32 physmem_map_index) +static uword +vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size) +{ + uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size; + alloc_size = CLIB_CACHE_LINE_ROUND (alloc_size); + + /* in case when we have even number of cachelines, we add one more for + * better cache occupancy */ + alloc_size |= CLIB_CACHE_LINE_BYTES; + + return alloc_size; +} + +u8 +vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size, + u32 physmem_map_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_pool_t *bp; vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); uword start = pointer_to_uword (m->base); uword size = (uword) m->n_pages << m->log2_page_size; - u32 alloc_size, n_alloc_per_page;; - - vec_validate_aligned (bm->buffer_pools, index, CLIB_CACHE_LINE_BYTES); - bp = vec_elt_at_index (bm->buffer_pools, index); + uword i, j; + u32 alloc_size, n_alloc_per_page; - if (bp->start) - return clib_error_return (0, "buffer with index %u already exists", - index); + if (vec_len (bm->buffer_pools) >= 255) + return ~0; - if (index >= 255) - return clib_error_return (0, "buffer index must be < 255", index); + vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_LOG2_CACHE_LINE_BYTES); if (bm->buffer_mem_size == 0) { @@ -538,43 +546,46 @@ vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name, bp->data_size = data_size; bp->numa_node = m->numa_node; - vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1, CLIB_CACHE_LINE_BYTES); - alloc_size = data_size + sizeof (vlib_buffer_t) + bm->ext_hdr_size; + alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size); n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size; /* preallocate buffer indices memory */ - vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page, - CLIB_CACHE_LINE_BYTES); - vec_reset_length (bp->buffers); + bp->n_buffers = m->n_pages * n_alloc_per_page; + bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32), + CLIB_CACHE_LINE_BYTES); clib_spinlock_init (&bp->lock); - while (1) - { - u8 *p; - u32 bi; - - p = vlib_physmem_alloc_from_map (vm, bp->physmem_map_index, alloc_size, - CLIB_CACHE_LINE_BYTES); + for (j = 0; j < m->n_pages; j++) + for (i = 0; i < n_alloc_per_page; i++) + { + u8 *p; + u32 bi; - if (p == 0) - break; + p = m->base + (j << m->log2_page_size) + i * alloc_size; + p += bm->ext_hdr_size; - p += bm->ext_hdr_size; + /* + * Waste 1 buffer (maximum) so that 0 is never a valid buffer index. + * Allows various places to ASSERT (bi != 0). Much easier + * than debugging downstream crashes in successor nodes. + */ + if (p == m->base) + continue; - vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template); + vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template); - bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p); + bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p); - vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES); - vlib_get_buffer (vm, bi); + bp->buffers[bp->n_avail++] = bi; - bp->n_buffers += 1; - } + vlib_get_buffer (vm, bi); + } - return 0; + return bp->index; } static u8 * @@ -592,14 +603,14 @@ format_vlib_buffer_pool (u8 * s, va_list * va) /* *INDENT-OFF* */ vec_foreach (bpt, bp->threads) - cached += vec_len (bpt->cached_buffers); + cached += bpt->n_cached; /* *INDENT-ON* */ s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", bp->name, bp->index, bp->numa_node, bp->data_size + sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size, - bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached, - bp->n_buffers - vec_len (bp->buffers) - cached); + bp->data_size, bp->n_buffers, bp->n_avail, cached, + bp->n_buffers - bp->n_avail - cached); return s; } @@ -639,7 +650,7 @@ vlib_buffer_worker_init (vlib_main_t * vm) vec_foreach (bp, bm->buffer_pools) { clib_spinlock_lock (&bp->lock); - vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1, CLIB_CACHE_LINE_BYTES); clib_spinlock_unlock (&bp->lock); } @@ -651,66 +662,180 @@ vlib_buffer_worker_init (vlib_main_t * vm) VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init); static clib_error_t * -vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node) +vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node, + u32 * physmem_map_index, + clib_mem_page_sz_t log2_page_size, + u8 unpriv) { vlib_buffer_main_t *bm = vm->buffer_main; + u32 buffers_per_numa = bm->buffers_per_numa; clib_error_t *error; - u32 physmem_map_index; + u32 buffer_size; uword n_pages, pagesize; - u32 buffers_per_numa; - u32 buffer_size = CLIB_CACHE_LINE_ROUND (bm->ext_hdr_size + - sizeof (vlib_buffer_t) + - VLIB_BUFFER_DATA_SIZE); - u8 *name; + u8 *name = 0; - pagesize = clib_mem_get_default_hugepage_size (); - name = format (0, "buffers-numa-%d%c", numa_node, 0); + ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN); + + pagesize = clib_mem_page_bytes (log2_page_size); + buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size, + vlib_buffer_get_default_data_size + (vm)); + if (buffer_size > pagesize) + return clib_error_return (0, "buffer size (%llu) is greater than page " + "size (%llu)", buffer_size, pagesize); - buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : - VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; + if (buffers_per_numa == 0) + buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; -retry: + name = format (0, "buffers-numa-%d%c", numa_node, 0); n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1; error = vlib_physmem_shared_map_create (vm, (char *) name, n_pages * pagesize, min_log2 (pagesize), numa_node, - &physmem_map_index); + physmem_map_index); + vec_free (name); + return error; +} + +static clib_error_t * +vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node, + u8 * index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + u32 physmem_map_index; + clib_error_t *error; + u8 *name = 0; - if (error && pagesize != clib_mem_get_page_size ()) + if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN) { - vlib_log_warn (bm->log_default, "%U", format_clib_error, error); + error = vlib_buffer_main_init_numa_alloc (vm, numa_node, + &physmem_map_index, + CLIB_MEM_PAGE_SZ_DEFAULT_HUGE, + 0 /* unpriv */ ); + if (!error) + goto buffer_pool_create; + + /* If alloc failed, retry without hugepages */ + vlib_log_warn (bm->log_default, + "numa[%u] falling back to non-hugepage backed " + "buffer pool (%U)", numa_node, format_clib_error, error); clib_error_free (error); - vlib_log_warn (bm->log_default, "falling back to non-hugepage " - "backed buffer pool"); - pagesize = clib_mem_get_page_size (); - buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : - VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV; - goto retry; - } + error = vlib_buffer_main_init_numa_alloc (vm, numa_node, + &physmem_map_index, + CLIB_MEM_PAGE_SZ_DEFAULT, + 1 /* unpriv */ ); + } + else + error = vlib_buffer_main_init_numa_alloc (vm, numa_node, + &physmem_map_index, + bm->log2_page_size, + 0 /* unpriv */ ); if (error) return error; - vec_reset_length (name); +buffer_pool_create: name = format (name, "default-numa-%d%c", numa_node, 0); + *index = vlib_buffer_pool_create (vm, (char *) name, + vlib_buffer_get_default_data_size (vm), + physmem_map_index); + + if (*index == (u8) ~ 0) + error = clib_error_return (0, "maximum number of buffer pools reached"); + vec_free (name); + + + return error; +} - return vlib_buffer_pool_create (vm, numa_node, (char *) name, - VLIB_BUFFER_DATA_SIZE, physmem_map_index); +void +vlib_buffer_main_alloc (vlib_main_t * vm) +{ + vlib_buffer_main_t *bm; + + if (vm->buffer_main) + return; + + vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0])); + clib_memset (vm->buffer_main, 0, sizeof (bm[0])); + bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; +} + +static u32 +buffer_get_cached (vlib_buffer_pool_t * bp) +{ + u32 cached = 0; + vlib_buffer_pool_thread_t *bpt; + + clib_spinlock_lock (&bp->lock); + + /* *INDENT-OFF* */ + vec_foreach (bpt, bp->threads) + cached += bpt->n_cached; + /* *INDENT-ON* */ + + clib_spinlock_unlock (&bp->lock); + + return cached; +} + +static vlib_buffer_pool_t * +buffer_get_by_index (vlib_buffer_main_t * bm, u32 index) +{ + vlib_buffer_pool_t *bp; + if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index) + return 0; + bp = vec_elt_at_index (bm->buffer_pools, index); + + return bp; +} + +static void +buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp); +} + +static void +buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e, + u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = bp->n_avail; +} + +static void +buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index); + if (!bp) + return; + + e->value = buffer_get_cached (bp); } clib_error_t * -vlib_buffer_main_init (struct vlib_main_t *vm) +vlib_buffer_main_init (struct vlib_main_t * vm) { vlib_buffer_main_t *bm; clib_error_t *err; - clib_bitmap_t *bmp = 0; + clib_bitmap_t *bmp = 0, *bmp_has_memory = 0; u32 numa_node; + vlib_buffer_pool_t *bp; + u8 *name = 0, first_valid_buffer_pool_index = ~0; - if (vm->buffer_main == 0) - { - vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); - clib_memset (vm->buffer_main, 0, sizeof (bm[0])); - } + vlib_buffer_main_alloc (vm); bm = vm->buffer_main; bm->log_default = vlib_log_register_class ("buffer", 0); @@ -718,26 +843,82 @@ vlib_buffer_main_init (struct vlib_main_t *vm) clib_spinlock_init (&bm->buffer_known_hash_lockp); - err = clib_sysfs_read ("/sys/devices/system/node/possible", "%U", - unformat_bitmap_list, &bmp); - if (err) + if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U", + unformat_bitmap_list, &bmp))) + clib_error_free (err); + + if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U", + unformat_bitmap_list, &bmp_has_memory))) + clib_error_free (err); + + if (bmp && bmp_has_memory) + bmp = clib_bitmap_and (bmp, bmp_has_memory); + + /* no info from sysfs, assuming that only numa 0 exists */ + if (bmp == 0) + bmp = clib_bitmap_set (bmp, 0, 1); + + if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES) + clib_panic ("system have more than %u NUMA nodes", + VLIB_BUFFER_MAX_NUMA_NODES); + + /* *INDENT-OFF* */ + clib_bitmap_foreach (numa_node, bmp) + { + u8 *index = bm->default_buffer_pool_index_for_numa + numa_node; + index[0] = ~0; + if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index))) + { + clib_error_report (err); + clib_error_free (err); + continue; + } + + if (first_valid_buffer_pool_index == 0xff) + first_valid_buffer_pool_index = index[0]; + } + /* *INDENT-ON* */ + + if (first_valid_buffer_pool_index == (u8) ~ 0) { - /* no info from sysfs, assuming that only numa 0 exists */ - clib_error_free (err); - bmp = clib_bitmap_set (bmp, 0, 1); + err = clib_error_return (0, "failed to allocate buffer pool(s)"); + goto done; } /* *INDENT-OFF* */ - clib_bitmap_foreach (numa_node, bmp, { - if ((err = vlib_buffer_main_init_numa_node(vm, numa_node))) - goto done; - }); + clib_bitmap_foreach (numa_node, bmp) + { + if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0) + bm->default_buffer_pool_index_for_numa[numa_node] = + first_valid_buffer_pool_index; + } /* *INDENT-ON* */ - bm->n_numa_nodes = clib_bitmap_last_set (bmp) + 1; + vec_foreach (bp, bm->buffer_pools) + { + if (bp->n_buffers == 0) + continue; + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/cached%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_cached_fn, + bp - bm->buffer_pools); + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/used%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_used_fn, + bp - bm->buffer_pools); + + vec_reset_length (name); + name = format (name, "/buffer-pools/%s/available%c", bp->name, 0); + stat_segment_register_gauge (name, buffer_gauges_update_available_fn, + bp - bm->buffer_pools); + } done: vec_free (bmp); + vec_free (bmp_has_memory); + vec_free (name); return err; } @@ -746,18 +927,21 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) { vlib_buffer_main_t *bm; - if (vm->buffer_main == 0) - { - vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); - clib_memset (vm->buffer_main, 0, sizeof (bm[0])); - } + vlib_buffer_main_alloc (vm); bm = vm->buffer_main; + bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa)) ; + else if (unformat (input, "page-size %U", unformat_log2_page_size, + &bm->log2_page_size)) + ; + else if (unformat (input, "default data-size %u", + &bm->default_data_size)) + ; else return unformat_parse_error (input); } @@ -768,6 +952,24 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers"); +#if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0 +u32 +vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers) +{ + f64 r; + + r = random_f64 (&vm->buffer_alloc_success_seed); + + /* Fail this request? */ + if (r > vm->buffer_alloc_success_rate) + n_buffers--; + /* 5% chance of returning nothing at all */ + if (r > vm->buffer_alloc_success_rate && r > 0.95) + n_buffers = 0; + + return n_buffers; +} +#endif /** @endcond */ /*