X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Ffifo_segment.c;h=ec6f082398e824549e90772c80005b5cca188157;hb=8e755a16a71c55555f12381c8a12e22ae7138536;hp=58e392d72466a22de6eb8921953268da0dd9686e;hpb=8122cc2ac5c9968c14a2570b8870a4ff7a4893c4;p=vpp.git diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 58e392d7246..ec6f082398e 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -15,6 +15,12 @@ #include +static char *fifo_segment_mem_status_strings[] = { +#define _(sym,str) str, + foreach_segment_mem_status +#undef _ +}; + /** * Fifo segment free space * @@ -26,11 +32,11 @@ * @return number of free bytes */ static uword -fs_free_space (fifo_segment_t * fs) +fsh_free_space (fifo_segment_header_t * fsh) { struct dlmallinfo dlminfo; - dlminfo = mspace_mallinfo (fs->ssvm.sh->heap); + dlminfo = mspace_mallinfo (fsh->ssvm_sh->heap); return dlminfo.fordblks; } @@ -48,10 +54,29 @@ fsh_n_free_bytes (fifo_segment_header_t * fsh) } static inline void -fsh_update_free_btes (fifo_segment_header_t * fsh) +fsh_update_free_bytes (fifo_segment_header_t * fsh) +{ + clib_atomic_store_rel_n (&fsh->n_free_bytes, fsh_free_space (fsh)); +} + +static inline void +fsh_cached_bytes_add (fifo_segment_header_t * fsh, int size) +{ + clib_atomic_fetch_add_rel (&fsh->n_cached_bytes, size); +} + +static inline void +fsh_cached_bytes_sub (fifo_segment_header_t * fsh, int size) { - clib_atomic_store_rel_n (&fsh->n_free_bytes, - fs_free_space (fsh->ssvm_sh->heap)); + clib_atomic_fetch_sub_rel (&fsh->n_cached_bytes, size); +} + +static inline uword +fsh_n_cached_bytes (fifo_segment_header_t * fsh) +{ + uword n_cached = clib_atomic_load_relax_n (&fsh->n_cached_bytes); + ASSERT (n_cached >= 0); + return n_cached; } static void @@ -68,7 +93,7 @@ fsh_check_mem (fifo_segment_header_t * fsh) return; fsh->flags |= FIFO_SEGMENT_F_MEM_LIMIT; - fsh_update_free_btes (fsh); + fsh_update_free_bytes (fsh); } static inline fifo_segment_slice_t * @@ -116,7 +141,7 @@ fifo_segment_init (fifo_segment_t * fs) fsh->ssvm_sh = fs->ssvm.sh; fsh->n_slices = fs->n_slices; - max_fifo = clib_min ((fs_free_space (fs) - 4096) / 2, + max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE); fsh->max_log2_chunk_size = max_log2 (max_fifo); @@ -128,11 +153,13 @@ fifo_segment_init (fifo_segment_t * fs) { fss = fsh_slice_get (fsh, i); vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0); + clib_spinlock_init (&fss->chunk_lock); } ssvm_pop_heap (oldheap); - fsh->n_free_bytes = fs_free_space (fs); + fsh->n_free_bytes = fsh_free_space (fsh); + fsh->n_cached_bytes = 0; max_chunks = fsh->n_free_bytes / FIFO_SEGMENT_MIN_FIFO_SIZE; fsh->n_reserved_bytes = (max_chunks / 4) * sizeof (rb_node_t); sh->ready = 1; @@ -244,6 +271,8 @@ fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, static inline u32 fs_freelist_for_size (u32 size) { + if (PREDICT_FALSE (size < FIFO_SEGMENT_MIN_FIFO_SIZE)) + return 0; return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; } @@ -265,8 +294,7 @@ fs_chunk_size_is_valid (fifo_segment_header_t * fsh, u32 size) } static svm_fifo_t * -fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, - u32 fl_index, u32 data_bytes) +fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, u32 fl_index) { svm_fifo_chunk_t *c; svm_fifo_t *f; @@ -279,9 +307,8 @@ fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, fss->free_fifos = f->next; fss->free_chunks[fl_index] = c->next; - c->next = c; + c->next = 0; c->start_byte = 0; - c->length = data_bytes; memset (f, 0, sizeof (*f)); f->start_chunk = c; f->end_chunk = c; @@ -290,6 +317,69 @@ fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, return f; } +svm_fifo_chunk_t * +fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, u32 data_bytes) +{ + u32 fl_index, fl_size, n_alloc = 0, req_bytes = data_bytes; + svm_fifo_chunk_t *c, *first = 0, *next; + + fl_index = fs_freelist_for_size (req_bytes); + if (fl_index > 0) + fl_index -= 1; + + fl_size = fs_freelist_index_to_size (fl_index); + + while (req_bytes) + { + c = fss->free_chunks[fl_index]; + if (c) + { + fss->free_chunks[fl_index] = c->next; + c->next = first; + first = c; + n_alloc += fl_size; + req_bytes -= clib_min (fl_size, req_bytes); + } + else + { + /* Failed to allocate with smaller chunks */ + if (fl_index == 0) + { + /* free all chunks if any allocated */ + c = first; + while (c) + { + fl_index = fs_freelist_for_size (c->length); + fl_size = fs_freelist_index_to_size (fl_index); + next = c->next; + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; + fss->n_fl_chunk_bytes += fl_size; + c = next; + } + n_alloc = 0; + first = 0; + fl_index = fs_freelist_for_size (data_bytes); + if (fss->free_chunks[fl_index + 1]) + { + fl_index += 1; + fl_size = fs_freelist_index_to_size (fl_index); + continue; + } + + return 0; + } + fl_index -= 1; + fl_size = fl_size >> 1; + } + } + + fss->n_fl_chunk_bytes -= n_alloc; + fsh_cached_bytes_sub (fsh, n_alloc); + return first; +} + static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, @@ -315,7 +405,10 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, fss->free_fifos = f->next; } - fl_index = fs_freelist_for_size (data_bytes) - 1; + fl_index = fs_freelist_for_size (data_bytes); + if (fl_index > 0) + fl_index -= 1; + fl_size = fs_freelist_index_to_size (fl_index); while (data_bytes) @@ -329,38 +422,111 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, c->next = first; first = c; n_alloc += fl_size; - c->length = clib_min (fl_size, data_bytes); data_bytes -= c->length; } else { - ASSERT (fl_index > 0); + /* Failed to allocate with smaller chunks */ + if (fl_index == 0) + { + /* free all chunks if any allocated */ + c = first; + while (c) + { + fl_index = fs_freelist_for_size (c->length); + fl_size = fs_freelist_index_to_size (fl_index); + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; + fss->n_fl_chunk_bytes += fl_size; + n_alloc -= fl_size; + data_bytes += fl_size; + } + first = last = 0; + fl_index = fs_freelist_for_size (data_bytes); + if (fss->free_chunks[fl_index + 1]) + { + fl_index += 1; + fl_size = fs_freelist_index_to_size (fl_index); + continue; + } + + f->next = fss->free_fifos; + fss->free_fifos = f; + return 0; + } fl_index -= 1; fl_size = fl_size >> 1; } } + f->start_chunk = first; f->end_chunk = last; - last->next = first; fss->n_fl_chunk_bytes -= n_alloc; + fsh_cached_bytes_sub (fsh, n_alloc); return f; } +static int +fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, + u32 fl_index, u32 batch_size) +{ + u32 rounded_data_size; + svm_fifo_chunk_t *c; + void *oldheap; + uword size; + u8 *cmem; + int i; + + rounded_data_size = fs_freelist_index_to_size (fl_index); + size = (uword) (sizeof (*c) + rounded_data_size) * batch_size; + + oldheap = ssvm_push_heap (fsh->ssvm_sh); + cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, + 0 /* align_offset */ , + 0 /* os_out_of_memory */ ); + ssvm_pop_heap (oldheap); + + /* Out of space.. */ + if (cmem == 0) + return -1; + + /* Carve fifo + chunk space */ + for (i = 0; i < batch_size; i++) + { + c = (svm_fifo_chunk_t *) cmem; + c->start_byte = 0; + c->length = rounded_data_size; + c->enq_rb_index = RBTREE_TNIL_INDEX; + c->deq_rb_index = RBTREE_TNIL_INDEX; + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; + cmem += sizeof (*c) + rounded_data_size; + } + + fss->n_fl_chunk_bytes += batch_size * rounded_data_size; + fsh_cached_bytes_add (fsh, batch_size * rounded_data_size); + fsh_free_bytes_sub (fsh, size); + + return 0; +} + static int fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 fl_index, u32 batch_size) { - u32 size, hdrs, rounded_data_size; + u32 hdrs, rounded_data_size; svm_fifo_chunk_t *c; svm_fifo_t *f; void *oldheap; + uword size; u8 *fmem; int i; rounded_data_size = fs_freelist_index_to_size (fl_index); hdrs = sizeof (*f) + sizeof (*c); - size = (hdrs + rounded_data_size) * batch_size; + size = (uword) (hdrs + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -382,12 +548,15 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, c = (svm_fifo_chunk_t *) (fmem + sizeof (*f)); c->start_byte = 0; c->length = rounded_data_size; + c->enq_rb_index = RBTREE_TNIL_INDEX; + c->deq_rb_index = RBTREE_TNIL_INDEX; c->next = fss->free_chunks[fl_index]; fss->free_chunks[fl_index] = c; fmem += hdrs + rounded_data_size; } fss->n_fl_chunk_bytes += batch_size * rounded_data_size; + fsh_cached_bytes_add (fsh, batch_size * rounded_data_size); fsh_free_bytes_sub (fsh, size); return 0; @@ -409,16 +578,21 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 fifo_sz, fl_index; svm_fifo_t *f = 0; uword n_free_bytes; + u32 min_size; - fl_index = fs_freelist_for_size (data_bytes); + min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096); + fl_index = fs_freelist_for_size (min_size); fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t); - fifo_sz += 1 << max_log2 (data_bytes); + fifo_sz += 1 << max_log2 (min_size); if (fss->free_fifos && fss->free_chunks[fl_index]) { - f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes); + f = fs_try_alloc_fifo_freelist (fss, fl_index); if (f) - goto done; + { + fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index)); + goto done; + } } fsh_check_mem (fsh); @@ -429,13 +603,15 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, FIFO_SEGMENT_ALLOC_BATCH_SIZE)) goto done; - f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes); + f = fs_try_alloc_fifo_freelist (fss, fl_index); + if (f) + fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index)); goto done; } if (fifo_sz <= n_free_bytes) { void *oldheap = ssvm_push_heap (fsh->ssvm_sh); - f = svm_fifo_create (data_bytes); + f = svm_fifo_alloc (min_size); ssvm_pop_heap (oldheap); if (f) { @@ -448,9 +624,111 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, done: + if (f) + f->fs_hdr = fsh; return f; } +svm_fifo_chunk_t * +fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size) +{ + fifo_segment_slice_t *fss; + svm_fifo_chunk_t *c; + void *oldheap; + int fl_index; + uword n_free; + + fl_index = fs_freelist_for_size (chunk_size); + fss = fsh_slice_get (fsh, slice_index); + + clib_spinlock_lock (&fss->chunk_lock); + + c = fss->free_chunks[fl_index]; + + if (c) + { + fss->free_chunks[fl_index] = c->next; + c->next = 0; + fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); + fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index)); + } + else if (chunk_size <= (n_free = fsh_n_free_bytes (fsh))) + { + fsh_check_mem (fsh); + + chunk_size = fs_freelist_index_to_size (fl_index); + if (n_free < chunk_size) + goto done; + + oldheap = ssvm_push_heap (fsh->ssvm_sh); + c = svm_fifo_chunk_alloc (chunk_size); + ssvm_pop_heap (oldheap); + + if (!c) + goto done; + + fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c)); + } + else if (chunk_size <= fss->n_fl_chunk_bytes) + { + c = fs_try_alloc_multi_chunk (fsh, fss, chunk_size); + } + else if (chunk_size <= fss->n_fl_chunk_bytes + n_free) + { + u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE; + u32 batch; + + fsh_check_mem (fsh); + batch = (chunk_size - fss->n_fl_chunk_bytes) / min_size; + batch = clib_min (batch + 1, n_free / min_size); + if (!fsh_try_alloc_chunk_batch (fsh, fss, 0, batch)) + c = fs_try_alloc_multi_chunk (fsh, fss, chunk_size); + } + +done: + + clib_spinlock_unlock (&fss->chunk_lock); + + return c; +} + +static void +fsh_slice_collect_chunks (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, svm_fifo_chunk_t * c) +{ + svm_fifo_chunk_t *next; + int fl_index; + u32 n_collect = 0; + + clib_spinlock_lock (&fss->chunk_lock); + + while (c) + { + next = c->next; + fl_index = fs_freelist_for_size (c->length); + c->next = fss->free_chunks[fl_index]; + c->enq_rb_index = RBTREE_TNIL_INDEX; + c->deq_rb_index = RBTREE_TNIL_INDEX; + fss->free_chunks[fl_index] = c; + n_collect += fs_freelist_index_to_size (fl_index); + c = next; + } + + fss->n_fl_chunk_bytes += n_collect; + fsh_cached_bytes_add (fsh, n_collect); + + clib_spinlock_unlock (&fss->chunk_lock); +} + +void +fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index, + svm_fifo_chunk_t * c) +{ + fifo_segment_slice_t *fss; + fss = fsh_slice_get (fsh, slice_index); + fsh_slice_collect_chunks (fsh, fss, c); +} + /** * Allocate fifo in fifo segment */ @@ -471,17 +749,8 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, f->slice_index = slice_index; - /* (re)initialize the fifo, as in svm_fifo_create */ svm_fifo_init (f, data_bytes); - /* Initialize chunks and rbtree for multi-chunk fifos */ - if (f->start_chunk->next != f->start_chunk) - { - void *oldheap = ssvm_push_heap (fsh->ssvm_sh); - svm_fifo_init_chunks (f); - ssvm_pop_heap (oldheap); - } - /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track @@ -495,7 +764,14 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, } fss->fifos = f; f->flags |= SVM_FIFO_F_LL_TRACKED; + + svm_fifo_init_ooo_lookup (f, 0 /* ooo enq */ ); + } + else + { + svm_fifo_init_ooo_lookup (f, 1 /* ooo deq */ ); } + fsh_active_fifos_update (fsh, 1); done: @@ -509,10 +785,7 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh = fs->h; - svm_fifo_chunk_t *cur, *next; fifo_segment_slice_t *fss; - void *oldheap; - int fl_index; ASSERT (f->refcnt > 0); @@ -539,27 +812,13 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) fss->free_fifos = f; /* Free fifo chunks */ - cur = f->start_chunk; - do - { - next = cur->next; - fl_index = fs_freelist_for_size (cur->length); - ASSERT (fl_index < vec_len (fss->free_chunks)); - cur->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = cur; - fss->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index); - cur = next; - } - while (cur != f->start_chunk); + fsh_slice_collect_chunks (fsh, fss, f->start_chunk); - f->start_chunk = f->end_chunk = f->new_chunks = 0; + f->start_chunk = f->end_chunk = 0; f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; - oldheap = ssvm_push_heap (fsh->ssvm_sh); - svm_fifo_free_chunk_lookup (f); - ssvm_pop_heap (oldheap); - /* not allocated on segment heap */ + svm_fifo_free_chunk_lookup (f); svm_fifo_free_ooo_data (f); if (CLIB_DEBUG) @@ -579,12 +838,12 @@ fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, fifo_segment_slice_t *fss; svm_fifo_t *f; void *oldheap; - u32 size; + uword size; u8 *fmem; int i; fss = fsh_slice_get (fsh, slice_index); - size = (sizeof (*f)) * batch_size; + size = (uword) (sizeof (*f)) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -615,11 +874,12 @@ int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, u32 chunk_size, u32 batch_size) { - u32 size, rounded_data_size, fl_index; fifo_segment_header_t *fsh = fs->h; + u32 rounded_data_size, fl_index; fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; void *oldheap; + uword size; u8 *cmem; int i; @@ -631,7 +891,7 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, fl_index = fs_freelist_for_size (chunk_size); rounded_data_size = fs_freelist_index_to_size (fl_index); - size = (sizeof (*c) + rounded_data_size) * batch_size; + size = (uword) (sizeof (*c) + rounded_data_size) * batch_size; oldheap = ssvm_push_heap (fsh->ssvm_sh); cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, @@ -654,6 +914,7 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, c->next = fss->free_chunks[fl_index]; fss->free_chunks[fl_index] = c; cmem += sizeof (*c) + rounded_data_size; + fsh_cached_bytes_add (fsh, rounded_data_size); } fss->n_fl_chunk_bytes += batch_size * rounded_data_size; @@ -671,10 +932,10 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 * n_fifo_pairs) { u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc; + u32 hdrs, pairs_per_slice, alloc_now; fifo_segment_header_t *fsh = fs->h; int rx_fl_index, tx_fl_index, i; fifo_segment_slice_t *fss; - u32 hdrs, pairs_per_slice; uword space_available; /* Parameter check */ @@ -702,10 +963,11 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, /* Calculate space requirements */ pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size; - space_available = fs_free_space (fs); + space_available = fsh_free_space (fsh); pairs_to_alloc = space_available / pair_size; pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs); pairs_per_slice = pairs_to_alloc / fs->n_slices; + pairs_per_slice += pairs_to_alloc % fs->n_slices ? 1 : 0; if (!pairs_per_slice) return; @@ -713,88 +975,15 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, for (i = 0; i < fs->n_slices; i++) { fss = fsh_slice_get (fsh, i); - if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, pairs_to_alloc)) - clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc); - if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, pairs_to_alloc)) - clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc); - } - - /* Account for the pairs allocated */ - *n_fifo_pairs -= pairs_per_slice * fs->n_slices; -} - -int -fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) -{ - fifo_segment_header_t *fsh = fs->h; - fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - void *oldheap; - int fl_index; - - fl_index = fs_freelist_for_size (chunk_size); - fss = fsh_slice_get (fsh, f->slice_index); - - oldheap = ssvm_push_heap (fsh->ssvm_sh); - - c = fss->free_chunks[fl_index]; - - if (!c) - { - fsh_check_mem (fsh); - if (fsh_n_free_bytes (fsh) < chunk_size) - { - ssvm_pop_heap (oldheap); - return -1; - } - - c = svm_fifo_chunk_alloc (chunk_size); - if (!c) - { - ssvm_pop_heap (oldheap); - return -1; - } - fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c)); - } - else - { - fss->free_chunks[fl_index] = c->next; - c->next = 0; - fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); + alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs); + if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now)) + clib_warning ("rx prealloc failed: pairs %u", alloc_now); + if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now)) + clib_warning ("tx prealloc failed: pairs %u", alloc_now); + + /* Account for the pairs allocated */ + *n_fifo_pairs -= alloc_now; } - - svm_fifo_add_chunk (f, c); - - ssvm_pop_heap (oldheap); - return 0; -} - -int -fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f) -{ - fifo_segment_header_t *fsh = fs->h; - svm_fifo_chunk_t *cur, *next; - fifo_segment_slice_t *fss; - void *oldheap; - int fl_index; - - oldheap = ssvm_push_heap (fsh->ssvm_sh); - cur = svm_fifo_collect_chunks (f); - - fss = fsh_slice_get (fsh, f->slice_index); - - while (cur) - { - next = cur->next; - fl_index = fs_freelist_for_size (cur->length); - cur->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = cur; - cur = next; - } - - ssvm_pop_heap (oldheap); - - return 0; } /** @@ -902,8 +1091,25 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) void fifo_segment_update_free_bytes (fifo_segment_t * fs) { - fifo_segment_header_t *fsh = fs->h; - clib_atomic_store_rel_n (&fsh->n_free_bytes, fs_free_space (fs)); + fsh_update_free_bytes (fs->h); +} + +uword +fifo_segment_size (fifo_segment_t * fs) +{ + return fs->ssvm.ssvm_size; +} + +u8 +fsh_has_reached_mem_limit (fifo_segment_header_t * fsh) +{ + return (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0; +} + +void +fsh_reset_mem_limit (fifo_segment_header_t * fsh) +{ + fsh->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT; } uword @@ -912,6 +1118,12 @@ fifo_segment_free_bytes (fifo_segment_t * fs) return fsh_n_free_bytes (fs->h); } +uword +fifo_segment_cached_bytes (fifo_segment_t * fs) +{ + return fsh_n_cached_bytes (fs->h); +} + uword fifo_segment_fl_chunk_bytes (fifo_segment_t * fs) { @@ -955,6 +1167,52 @@ fifo_segment_get_slice_fifo_list (fifo_segment_t * fs, u32 slice_index) return fss->fifos; } +u8 +fifo_segment_get_mem_usage (fifo_segment_t * fs) +{ + uword size, in_use; + + size = fifo_segment_size (fs); + in_use = + size - fifo_segment_free_bytes (fs) - fifo_segment_cached_bytes (fs); + return (in_use * 100) / size; +} + +fifo_segment_mem_status_t +fifo_segment_determine_status (fifo_segment_header_t * fsh, u8 usage) +{ + if (!fsh->high_watermark || !fsh->low_watermark) + return MEMORY_PRESSURE_NO_PRESSURE; + + /* once the no-memory is detected, the status continues + * until memory usage gets below the high watermark + */ + if (fsh_has_reached_mem_limit (fsh)) + { + if (usage >= fsh->high_watermark) + return MEMORY_PRESSURE_NO_MEMORY; + else + fsh_reset_mem_limit (fsh); + } + + if (usage >= fsh->high_watermark) + return MEMORY_PRESSURE_HIGH_PRESSURE; + + else if (usage >= fsh->low_watermark) + return MEMORY_PRESSURE_LOW_PRESSURE; + + return MEMORY_PRESSURE_NO_PRESSURE; +} + +fifo_segment_mem_status_t +fifo_segment_get_mem_status (fifo_segment_t * fs) +{ + fifo_segment_header_t *fsh = fs->h; + u8 usage = fifo_segment_get_mem_usage (fs); + + return fifo_segment_determine_status (fsh, usage); +} + u8 * format_fifo_segment_type (u8 * s, va_list * args) { @@ -984,6 +1242,7 @@ format_fifo_segment (u8 * s, va_list * args) int verbose __attribute__ ((unused)) = va_arg (*args, int); uword est_chunk_bytes, est_free_seg_bytes, free_chunks; uword chunk_bytes = 0, free_seg_bytes, chunk_size; + uword tracked_cached_bytes; fifo_segment_header_t *fsh; fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; @@ -991,14 +1250,11 @@ format_fifo_segment (u8 * s, va_list * args) char *address; size_t size; int i; + uword allocated, in_use; + f64 usage; + fifo_segment_mem_status_t mem_st; indent = format_get_indent (s) + 2; -#if USE_DLMALLOC == 0 - s = format (s, "%U segment heap: %U\n", format_white_space, indent, - format_mheap, fsh->ssvm_sh->heap, verbose); - s = format (s, "%U segment has %u active fifos\n", - format_white_space, indent, fifo_segment_num_fifos (fsh)); -#endif if (fs == 0) { @@ -1055,19 +1311,31 @@ format_fifo_segment (u8 * s, va_list * args) est_free_seg_bytes = fifo_segment_free_bytes (fs); fifo_segment_update_free_bytes (fs); free_seg_bytes = fifo_segment_free_bytes (fs); + tracked_cached_bytes = fifo_segment_cached_bytes (fs); + allocated = fifo_segment_size (fs); + in_use = fifo_segment_size (fs) - est_free_seg_bytes - tracked_cached_bytes; + usage = (100.0 * in_use) / allocated; + mem_st = fifo_segment_get_mem_status (fs); s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu)\n", format_white_space, indent + 2, format_memory_size, free_seg_bytes, free_seg_bytes, format_memory_size, est_free_seg_bytes, est_free_seg_bytes); - s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%lu)\n", - format_white_space, indent + 2, format_memory_size, chunk_bytes, - chunk_bytes, format_memory_size, est_chunk_bytes, - est_chunk_bytes); - s = format (s, "%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n", - format_white_space, indent + 2, format_memory_size, fifo_hdr, - fifo_hdr, format_memory_size, fsh->n_reserved_bytes, - fsh->n_reserved_bytes); + s = + format (s, + "%Uchunk free bytes: %U (%lu) estimated: %U (%lu) tracked: %U (%lu)\n", + format_white_space, indent + 2, format_memory_size, chunk_bytes, + chunk_bytes, format_memory_size, est_chunk_bytes, est_chunk_bytes, + format_memory_size, tracked_cached_bytes, tracked_cached_bytes); + s = + format (s, "%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n", + format_white_space, indent + 2, format_memory_size, fifo_hdr, + fifo_hdr, format_memory_size, fsh->n_reserved_bytes, + fsh->n_reserved_bytes); + s = + format (s, "%Usegment usage: %.2f%% (%U / %U) %s\n", format_white_space, + indent + 2, usage, format_memory_size, in_use, format_memory_size, + allocated, fifo_segment_mem_status_strings[mem_st]); s = format (s, "\n"); return s;