X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Ffifo_segment.c;h=58c11383e43e3419b8c73f81a5d10d7165756c56;hb=2c8e0023f91882e53f06eb99c901b97fe013f981;hp=95b88836ddbe226b7b1af5b70acda62b6256248c;hpb=62ddc030296e9f65c59cd25fa81112e0a09cf20d;p=vpp.git diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 95b88836ddb..58c11383e43 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -25,12 +25,12 @@ * @param fs fifo segment * @return number of free bytes */ -static u32 -fs_free_space (fifo_segment_t * fs) +static uword +fsh_free_space (fifo_segment_header_t * fsh) { struct dlmallinfo dlminfo; - dlminfo = mspace_mallinfo (fs->ssvm.sh->heap); + dlminfo = mspace_mallinfo (fsh->ssvm_sh->heap); return dlminfo.fordblks; } @@ -43,7 +43,31 @@ fsh_free_bytes_sub (fifo_segment_header_t * fsh, int size) static inline uword fsh_n_free_bytes (fifo_segment_header_t * fsh) { - return clib_atomic_load_relax_n (&fsh->n_free_bytes); + uword n_free = clib_atomic_load_relax_n (&fsh->n_free_bytes); + return n_free > fsh->n_reserved_bytes ? n_free - fsh->n_reserved_bytes : 0; +} + +static inline void +fsh_update_free_bytes (fifo_segment_header_t * fsh) +{ + clib_atomic_store_rel_n (&fsh->n_free_bytes, fsh_free_space (fsh)); +} + +static void +fsh_check_mem (fifo_segment_header_t * fsh) +{ + uword thresh; + + if (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) + return; + + thresh = clib_max (0.01 * fsh->ssvm_sh->ssvm_size, + 2 * fsh->n_reserved_bytes); + if (fsh->n_free_bytes > thresh) + return; + + fsh->flags |= FIFO_SEGMENT_F_MEM_LIMIT; + fsh_update_free_bytes (fsh); } static inline fifo_segment_slice_t * @@ -67,7 +91,7 @@ fifo_segment_init (fifo_segment_t * fs) fifo_segment_header_t *fsh; fifo_segment_slice_t *fss; ssvm_shared_header_t *sh; - u32 max_chunk_sz; + u32 max_chunk_sz, max_chunks; uword max_fifo; void *oldheap; int i; @@ -91,7 +115,7 @@ fifo_segment_init (fifo_segment_t * fs) fsh->ssvm_sh = fs->ssvm.sh; fsh->n_slices = fs->n_slices; - max_fifo = clib_min ((fs_free_space (fs) - 4096) / 2, + max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE); fsh->max_log2_chunk_size = max_log2 (max_fifo); @@ -107,7 +131,9 @@ fifo_segment_init (fifo_segment_t * fs) ssvm_pop_heap (oldheap); - fsh->n_free_bytes = fs_free_space (fs); + fsh->n_free_bytes = fsh_free_space (fsh); + max_chunks = fsh->n_free_bytes / FIFO_SEGMENT_MIN_FIFO_SIZE; + fsh->n_reserved_bytes = (max_chunks / 4) * sizeof (rb_node_t); sh->ready = 1; return (0); } @@ -288,7 +314,10 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, fss->free_fifos = f->next; } - fl_index = fs_freelist_for_size (data_bytes) - 1; + fl_index = fs_freelist_for_size (data_bytes); + if (fl_index > 0) + fl_index -= 1; + fl_size = fs_freelist_index_to_size (fl_index); while (data_bytes) @@ -307,11 +336,38 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, } else { - ASSERT (fl_index > 0); + /* Failed to allocate with smaller chunks */ + if (fl_index == 0) + { + /* free all chunks if any allocated */ + c = first; + while (c) + { + fl_index = fs_freelist_for_size (c->length); + fl_size = fs_freelist_index_to_size (fl_index); + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; + fss->n_fl_chunk_bytes += fl_size; + data_bytes += fl_size; + } + first = last = 0; + fl_index = fs_freelist_for_size (data_bytes); + if (fss->free_chunks[fl_index + 1]) + { + fl_index += 1; + fl_size = fs_freelist_index_to_size (fl_index); + continue; + } + + f->next = fss->free_fifos; + fss->free_fifos = f; + return 0; + } fl_index -= 1; fl_size = fl_size >> 1; } } + f->start_chunk = first; f->end_chunk = last; last->next = first; @@ -324,16 +380,17 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 fl_index, u32 batch_size) { - u32 size, hdrs, rounded_data_size; + u32 hdrs, rounded_data_size; svm_fifo_chunk_t *c; svm_fifo_t *f; void *oldheap; + uword size; u8 *fmem; int i; rounded_data_size = fs_freelist_index_to_size (fl_index); hdrs = sizeof (*f) + sizeof (*c); - size = (hdrs + rounded_data_size) * batch_size; + size = (uword) (hdrs + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -393,6 +450,8 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, if (f) goto done; } + + fsh_check_mem (fsh); n_free_bytes = fsh_n_free_bytes (fsh); if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < n_free_bytes) { @@ -550,12 +609,12 @@ fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, fifo_segment_slice_t *fss; svm_fifo_t *f; void *oldheap; - u32 size; + uword size; u8 *fmem; int i; fss = fsh_slice_get (fsh, slice_index); - size = (sizeof (*f)) * batch_size; + size = (uword) (sizeof (*f)) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -586,11 +645,12 @@ int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, u32 chunk_size, u32 batch_size) { - u32 size, rounded_data_size, fl_index; fifo_segment_header_t *fsh = fs->h; + u32 rounded_data_size, fl_index; fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; void *oldheap; + uword size; u8 *cmem; int i; @@ -602,7 +662,7 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, fl_index = fs_freelist_for_size (chunk_size); rounded_data_size = fs_freelist_index_to_size (fl_index); - size = (sizeof (*c) + rounded_data_size) * batch_size; + size = (uword) (sizeof (*c) + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -642,10 +702,10 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 * n_fifo_pairs) { u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc; + u32 hdrs, pairs_per_slice, alloc_now; fifo_segment_header_t *fsh = fs->h; int rx_fl_index, tx_fl_index, i; fifo_segment_slice_t *fss; - u32 hdrs, pairs_per_slice; uword space_available; /* Parameter check */ @@ -673,10 +733,11 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, /* Calculate space requirements */ pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size; - space_available = fs_free_space (fs); + space_available = fsh_free_space (fsh); pairs_to_alloc = space_available / pair_size; pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs); pairs_per_slice = pairs_to_alloc / fs->n_slices; + pairs_per_slice += pairs_to_alloc % fs->n_slices ? 1 : 0; if (!pairs_per_slice) return; @@ -684,14 +745,15 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, for (i = 0; i < fs->n_slices; i++) { fss = fsh_slice_get (fsh, i); - if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, pairs_to_alloc)) - clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc); - if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, pairs_to_alloc)) - clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc); + alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs); + if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now)) + clib_warning ("rx prealloc failed: pairs %u", alloc_now); + if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now)) + clib_warning ("tx prealloc failed: pairs %u", alloc_now); + + /* Account for the pairs allocated */ + *n_fifo_pairs -= alloc_now; } - - /* Account for the pairs allocated */ - *n_fifo_pairs -= pairs_per_slice * fs->n_slices; } int @@ -712,6 +774,13 @@ fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) if (!c) { + fsh_check_mem (fsh); + if (fsh_n_free_bytes (fsh) < chunk_size) + { + ssvm_pop_heap (oldheap); + return -1; + } + c = svm_fifo_chunk_alloc (chunk_size); if (!c) { @@ -866,11 +935,10 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) void fifo_segment_update_free_bytes (fifo_segment_t * fs) { - fifo_segment_header_t *fsh = fs->h; - clib_atomic_store_rel_n (&fsh->n_free_bytes, fs_free_space (fs)); + fsh_update_free_bytes (fs->h); } -u32 +uword fifo_segment_free_bytes (fifo_segment_t * fs) { return fsh_n_free_bytes (fs->h); @@ -943,11 +1011,11 @@ format_fifo_segment_type (u8 * s, va_list * args) u8 * format_fifo_segment (u8 * s, va_list * args) { - u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0, chunk_size; + u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0; fifo_segment_t *fs = va_arg (*args, fifo_segment_t *); int verbose __attribute__ ((unused)) = va_arg (*args, int); - u32 est_chunk_bytes, est_free_seg_bytes, free_chunks; - uword chunk_bytes = 0, free_seg_bytes; + uword est_chunk_bytes, est_free_seg_bytes, free_chunks; + uword chunk_bytes = 0, free_seg_bytes, chunk_size; fifo_segment_header_t *fsh; fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; @@ -957,12 +1025,6 @@ format_fifo_segment (u8 * s, va_list * args) int i; indent = format_get_indent (s) + 2; -#if USE_DLMALLOC == 0 - s = format (s, "%U segment heap: %U\n", format_white_space, indent, - format_mheap, fsh->ssvm_sh->heap, verbose); - s = format (s, "%U segment has %u active fifos\n", - format_white_space, indent, fifo_segment_num_fifos (fsh)); -#endif if (fs == 0) { @@ -1020,16 +1082,18 @@ format_fifo_segment (u8 * s, va_list * args) fifo_segment_update_free_bytes (fs); free_seg_bytes = fifo_segment_free_bytes (fs); - s = format (s, "\n%Useg free bytes: %U (%u) estimated: %U (%u)\n", + s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu)\n", format_white_space, indent + 2, format_memory_size, free_seg_bytes, free_seg_bytes, format_memory_size, est_free_seg_bytes, est_free_seg_bytes); - s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%u)\n", + s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%lu)\n", format_white_space, indent + 2, format_memory_size, chunk_bytes, chunk_bytes, format_memory_size, est_chunk_bytes, est_chunk_bytes); - s = format (s, "%Ufifo hdr free bytes: %U (%u)\n", format_white_space, - indent + 2, format_memory_size, fifo_hdr, fifo_hdr); + s = format (s, "%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n", + format_white_space, indent + 2, format_memory_size, fifo_hdr, + fifo_hdr, format_memory_size, fsh->n_reserved_bytes, + fsh->n_reserved_bytes); s = format (s, "\n"); return s;