X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Ffifo_segment.c;h=d5f62913082175887f0d3a29ae5086cd5669033e;hb=HEAD;hp=a5ffd4af2ed0d9ea0d1ae1f975323cecccbcaaae;hpb=8c4fa01d1360cd5315e671de96dfeff7dae246f5;p=vpp.git diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index a5ffd4af2ed..d5f62913082 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -14,6 +14,37 @@ */ #include +#include + +static inline void * +fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align) +{ + uword cur_pos, cur_pos_align, new_pos; + + cur_pos = clib_atomic_load_relax_n (&fsh->byte_index); + cur_pos_align = round_pow2_u64 (cur_pos, align); + size = round_pow2_u64 (size, align); + new_pos = cur_pos_align + size; + + if (new_pos >= fsh->max_byte_index) + return 0; + + while (!clib_atomic_cmp_and_swap_acq_relax (&fsh->byte_index, &cur_pos, + &new_pos, 1 /* weak */)) + { + cur_pos_align = round_pow2_u64 (cur_pos, align); + new_pos = cur_pos_align + size; + if (new_pos >= fsh->max_byte_index) + return 0; + } + return uword_to_pointer ((u8 *) fsh + cur_pos_align, void *); +} + +static inline void * +fsh_alloc (fifo_segment_header_t *fsh, uword size) +{ + return fsh_alloc_aligned (fsh, size, 8); +} static inline fifo_segment_slice_t * fsh_slice_get (fifo_segment_header_t * fsh, u32 slice_index) @@ -21,55 +52,35 @@ fsh_slice_get (fifo_segment_header_t * fsh, u32 slice_index) return &fsh->slices[slice_index]; } +static inline fifo_slice_private_t * +fs_slice_private_get (fifo_segment_t *fs, u32 slice_index) +{ + ASSERT (slice_index < fs->n_slices); + return &fs->slices[slice_index]; +} + static char *fifo_segment_mem_status_strings[] = { #define _(sym,str) str, foreach_segment_mem_status #undef _ }; -/** - * Fifo segment free space - * - * Queries the underlying memory manager, dlmalloc, for free space. Since this - * ends up walking the internal data structures, it should not be called - * indiscriminately. - * - * @param fs fifo segment - * @return number of free bytes - */ -static uword -fsh_free_space (fifo_segment_header_t * fsh) -{ - return clib_mem_get_heap_free_space (fsh->ssvm_sh->heap); -} - -static inline void -fsh_free_bytes_sub (fifo_segment_header_t * fsh, int size) -{ - clib_atomic_fetch_sub_rel (&fsh->n_free_bytes, size); -} - static inline uword fsh_n_free_bytes (fifo_segment_header_t * fsh) { - uword n_free = clib_atomic_load_relax_n (&fsh->n_free_bytes); - return n_free > fsh->n_reserved_bytes ? n_free - fsh->n_reserved_bytes : 0; -} - -static inline void -fsh_update_free_bytes (fifo_segment_header_t * fsh) -{ - clib_atomic_store_rel_n (&fsh->n_free_bytes, fsh_free_space (fsh)); + uword cur_pos = clib_atomic_load_relax_n (&fsh->byte_index); + ASSERT (fsh->max_byte_index > cur_pos); + return fsh->max_byte_index - cur_pos; } static inline void -fsh_cached_bytes_add (fifo_segment_header_t * fsh, int size) +fsh_cached_bytes_add (fifo_segment_header_t * fsh, uword size) { clib_atomic_fetch_add_rel (&fsh->n_cached_bytes, size); } static inline void -fsh_cached_bytes_sub (fifo_segment_header_t * fsh, int size) +fsh_cached_bytes_sub (fifo_segment_header_t * fsh, uword size) { clib_atomic_fetch_sub_rel (&fsh->n_cached_bytes, size); } @@ -94,13 +105,14 @@ fsh_n_active_fifos (fifo_segment_header_t * fsh) } static inline uword -fsh_virtual_mem (fifo_segment_header_t * fsh) +fs_virtual_mem (fifo_segment_t *fs) { + fifo_segment_header_t *fsh = fs->h; fifo_segment_slice_t *fss; uword total_vm = 0; int i; - for (i = 0; i < fsh->n_slices; i++) + for (i = 0; i < fs->n_slices; i++) { fss = fsh_slice_get (fsh, i); total_vm += clib_atomic_load_relax_n (&fss->virtual_mem); @@ -116,21 +128,165 @@ fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index, fss->virtual_mem += n_bytes; } +static inline int +fss_chunk_fl_index_is_valid (fifo_segment_slice_t *fss, u32 fl_index) +{ + return (fl_index < FS_CHUNK_VEC_LEN); +} + +#define FS_CL_HEAD_MASK 0xFFFFFFFFFFFF +#define FS_CL_HEAD_TMASK 0xFFFF000000000000 +#define FS_CL_HEAD_TINC (1ULL << 48) + +static svm_fifo_chunk_t * +fss_chunk_free_list_head (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 fl_index) +{ + fs_sptr_t headsp = clib_atomic_load_relax_n (&fss->free_chunks[fl_index]); + return fs_chunk_ptr (fsh, headsp & FS_CL_HEAD_MASK); +} + static void -fsh_check_mem (fifo_segment_header_t * fsh) +fss_chunk_free_list_push (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 fl_index, + svm_fifo_chunk_t *c) { - uword thresh; + fs_sptr_t old_head, new_head, csp; - if (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) - return; + csp = fs_chunk_sptr (fsh, c); + ASSERT (csp <= FS_CL_HEAD_MASK); + old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]); - thresh = clib_max (0.01 * fsh->ssvm_sh->ssvm_size, - 2 * fsh->n_reserved_bytes); - if (fsh->n_free_bytes > thresh) - return; + do + { + c->next = old_head & FS_CL_HEAD_MASK; + new_head = csp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK); + } + while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head, + &new_head, 0 /* weak */, __ATOMIC_RELEASE, + __ATOMIC_ACQUIRE)); +} + +static void +fss_chunk_free_list_push_list (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 fl_index, + svm_fifo_chunk_t *head, svm_fifo_chunk_t *tail) +{ + fs_sptr_t old_head, new_head, headsp; + + headsp = fs_chunk_sptr (fsh, head); + ASSERT (headsp <= FS_CL_HEAD_MASK); + old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]); + + do + { + tail->next = old_head & FS_CL_HEAD_MASK; + new_head = headsp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK); + } + while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head, + &new_head, 0 /* weak */, __ATOMIC_RELEASE, + __ATOMIC_ACQUIRE)); +} - fsh->flags |= FIFO_SEGMENT_F_MEM_LIMIT; - fsh_update_free_bytes (fsh); +static svm_fifo_chunk_t * +fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, + u32 fl_index) +{ + fs_sptr_t old_head, new_head; + svm_fifo_chunk_t *c; + + ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index)); + + old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]); + + /* Lock-free stacks are affected by ABA if a side allocates a chunk and + * shortly thereafter frees it. To circumvent that, reuse the upper bits + * of the head of the list shared pointer, i.e., offset to where the chunk + * is, as a tag. The tag is incremented with each push/pop operation and + * therefore collisions can only happen if an element is popped and pushed + * exactly after a complete wrap of the tag (16 bits). It's unlikely either + * of the sides will be descheduled for that long */ + do + { + if (!(old_head & FS_CL_HEAD_MASK)) + return 0; + c = fs_chunk_ptr (fsh, old_head & FS_CL_HEAD_MASK); + new_head = c->next + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK); + } + while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head, + &new_head, 0 /* weak */, __ATOMIC_RELEASE, + __ATOMIC_ACQUIRE)); + + return c; +} + +static void +fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, + svm_fifo_shared_t *sf) +{ + sf->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, sf); +} + +static void +fss_fifo_free_list_push_list (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, + svm_fifo_shared_t *head, svm_fifo_shared_t *tail) +{ + tail->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, head); +} + +svm_fifo_shared_t * +fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) +{ + svm_fifo_shared_t *sf; + sf = fs_ptr (fsh, fss->free_fifos); + fss->free_fifos = sf->next; + return sf; +} + +static inline void +pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f) +{ + if (pfss->active_fifos) + { + pfss->active_fifos->prev = f; + f->next = pfss->active_fifos; + } + pfss->active_fifos = f; +} + +static inline void +pfss_fifo_del_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f) +{ + if (f->flags & SVM_FIFO_F_LL_TRACKED) + { + if (f->prev) + f->prev->next = f->next; + else + pfss->active_fifos = f->next; + if (f->next) + f->next->prev = f->prev; + } +} + +static inline uword +fss_fl_chunk_bytes (fifo_segment_slice_t * fss) +{ + return clib_atomic_load_relax_n (&fss->n_fl_chunk_bytes); +} + +static inline void +fss_fl_chunk_bytes_add (fifo_segment_slice_t * fss, uword size) +{ + clib_atomic_fetch_add_relax (&fss->n_fl_chunk_bytes, size); +} + +static inline void +fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size) +{ + clib_atomic_fetch_sub_relax (&fss->n_fl_chunk_bytes, size); } /** @@ -139,54 +295,48 @@ fsh_check_mem (fifo_segment_header_t * fsh) int fifo_segment_init (fifo_segment_t * fs) { + u32 align = 8, offset = FIFO_SEGMENT_ALLOC_OVERHEAD, slices_sz, i; + uword max_fifo, seg_start, seg_sz; fifo_segment_header_t *fsh; - fifo_segment_slice_t *fss; ssvm_shared_header_t *sh; - u32 max_chunk_sz; - uword max_fifo; - void *oldheap; - int i; + void *seg_data; + /* TODO remove ssvm heap entirely */ sh = fs->ssvm.sh; - oldheap = ssvm_push_heap (sh); - /* - * Manually align the fifo segment header to sizeof(uword) = 8 bytes. - * Long story made short: the "process-private" fifo segment - * is allocated from the main heap, not mmapped. dlmalloc - * only guarantees 4-byte alignment, and on aarch64 - * the fsh can end up 4-byte but not 8-byte aligned. - * That eventually causes the atomic op in fifo_segment_update_free_bytes - * to backfire. - */ - fsh = clib_mem_alloc_aligned (sizeof (*fsh), sizeof (uword)); - clib_memset (fsh, 0, sizeof (*fsh)); - fs->h = sh->opaque[0] = fsh; + seg_data = (u8 *) sh + offset; + seg_sz = sh->ssvm_size - offset; + fs->n_slices = clib_max (fs->n_slices, 1); + slices_sz = sizeof (fifo_segment_slice_t) * fs->n_slices; + + seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align); + fsh = uword_to_pointer (seg_start, void *); + clib_mem_unpoison (fsh, seg_sz); + memset (fsh, 0, sizeof (*fsh) + slices_sz); - fsh->ssvm_sh = fs->ssvm.sh; + fsh->byte_index = sizeof (*fsh) + slices_sz; + fsh->max_byte_index = seg_sz; fsh->n_slices = fs->n_slices; - max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2, - FIFO_SEGMENT_MAX_FIFO_SIZE); - fsh->max_log2_chunk_size = max_log2 (max_fifo); + max_fifo = clib_min ((seg_sz - slices_sz) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE); + fsh->max_log2_fifo_size = min_log2 (max_fifo); + fsh->n_cached_bytes = 0; + fsh->n_reserved_bytes = fsh->byte_index; + fsh->start_byte_index = fsh->byte_index; + ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset); - fsh->slices = clib_mem_alloc (sizeof (*fss) * fs->n_slices); - clib_memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices); - max_chunk_sz = fsh->max_log2_chunk_size - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; + fs->max_byte_index = fsh->max_byte_index; + fs->h = fsh; + sh->opaque[0] = (void *) ((u8 *) fsh - (u8 *) fs->ssvm.sh); - for (i = 0; i < fs->n_slices; i++) - { - fss = fsh_slice_get (fsh, i); - vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0); - vec_validate_init_empty (fss->num_chunks, max_chunk_sz, 0); - clib_spinlock_init (&fss->chunk_lock); - } + /* Allow random offsets */ + fs->ssvm.sh->ssvm_va = 0; - ssvm_pop_heap (oldheap); + vec_validate (fs->slices, fs->n_slices - 1); + for (i = 0; i < fs->n_slices; i++) + fs->slices[i].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); - fsh->n_free_bytes = fsh_free_space (fsh); - fsh->n_cached_bytes = 0; - fsh->n_reserved_bytes = clib_min (0.01 * fsh->n_free_bytes, 256 << 10); sh->ready = 1; return (0); } @@ -231,15 +381,18 @@ fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) int fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { + fifo_segment_header_t *fsh; fifo_segment_t *fs; int rv; pool_get_zero (sm->segments, fs); + fs->fs_index = fs - sm->segments; + fs->sm_index = ~0; fs->ssvm.ssvm_size = a->segment_size; fs->ssvm.my_pid = getpid (); fs->ssvm.name = format (0, "%s%c", a->segment_name, 0); - fs->ssvm.requested_va = sm->next_baseva; + fs->ssvm.requested_va = 0; if (a->segment_type == SSVM_SEGMENT_MEMFD) fs->ssvm.fd = a->memfd_fd; else @@ -247,13 +400,21 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) if ((rv = ssvm_client_init (&fs->ssvm, a->segment_type))) { - _vec_len (fs) = vec_len (fs) - 1; + pool_put (sm->segments, fs); return (rv); } - /* Fish the segment header */ - fs->h = fs->ssvm.sh->opaque[0]; + /* Probably a segment without fifos */ + if (!fs->ssvm.sh->opaque[0]) + goto done; + + fsh = fs->h = (void *) fs->ssvm.sh + (uword) fs->ssvm.sh->opaque[0]; + fs->max_byte_index = fsh->max_byte_index; + vec_validate (fs->slices, 0); + fs->slices[0].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); +done: vec_add1 (a->new_segment_indices, fs - sm->segments); return (0); } @@ -261,6 +422,7 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s) { + fifo_segment_cleanup (s); ssvm_delete (&s->ssvm); clib_memset (s, 0xfe, sizeof (*s)); pool_put (sm->segments, s); @@ -278,6 +440,14 @@ fifo_segment_get_segment (fifo_segment_main_t * sm, u32 segment_index) return pool_elt_at_index (sm->segments, segment_index); } +fifo_segment_t * +fifo_segment_get_segment_if_valid (fifo_segment_main_t *sm, u32 segment_index) +{ + if (pool_is_free_index (sm->segments, segment_index)) + return 0; + return pool_elt_at_index (sm->segments, segment_index); +} + void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size) { @@ -298,7 +468,8 @@ fs_freelist_for_size (u32 size) { if (PREDICT_FALSE (size < FIFO_SEGMENT_MIN_FIFO_SIZE)) return 0; - return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; + return clib_min (max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE, + FS_CHUNK_VEC_LEN - 1); } static inline u32 @@ -314,32 +485,8 @@ fs_chunk_size_is_valid (fifo_segment_header_t * fsh, u32 size) * 4K minimum. It's not likely that anything good will happen * with a smaller FIFO. */ - return size >= FIFO_SEGMENT_MIN_FIFO_SIZE - && size <= (1ULL << fsh->max_log2_chunk_size); -} - -static svm_fifo_t * -fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, u32 fl_index) -{ - svm_fifo_chunk_t *c; - svm_fifo_t *f; - - f = fss->free_fifos; - c = fss->free_chunks[fl_index]; - - if (!f || !c) - return 0; - - fss->free_fifos = f->next; - fss->free_chunks[fl_index] = c->next; - c->next = 0; - c->start_byte = 0; - memset (f, 0, sizeof (*f)); - f->start_chunk = c; - f->end_chunk = c; - - fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); - return f; + return size >= FIFO_SEGMENT_MIN_FIFO_SIZE && + size <= (1ULL << fsh->max_log2_fifo_size); } svm_fifo_chunk_t * @@ -357,11 +504,10 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, while (req_bytes) { - c = fss->free_chunks[fl_index]; + c = fss_chunk_free_list_pop (fsh, fss, fl_index); if (c) { - fss->free_chunks[fl_index] = c->next; - c->next = first; + c->next = fs_chunk_sptr (fsh, first); first = c; n_alloc += fl_size; req_bytes -= clib_min (fl_size, req_bytes); @@ -371,28 +517,29 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, /* Failed to allocate with smaller chunks */ if (fl_index == 0) { - /* free all chunks if any allocated */ + /* Free all chunks if any allocated */ c = first; while (c) { fl_index = fs_freelist_for_size (c->length); - fl_size = fs_freelist_index_to_size (fl_index); - next = c->next; - c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; - fss->n_fl_chunk_bytes += fl_size; + next = fs_chunk_ptr (fsh, c->next); + fss_chunk_free_list_push (fsh, fss, fl_index, c); c = next; } n_alloc = 0; first = 0; - fl_index = fs_freelist_for_size (data_bytes); - if (fss->free_chunks[fl_index + 1]) + /* As last attempt, try allocating a chunk larger than + * the requested size, if possible */ + fl_index = fs_freelist_for_size (data_bytes) + 1; + if (!fss_chunk_fl_index_is_valid (fss, fl_index)) + return 0; + first = fss_chunk_free_list_pop (fsh, fss, fl_index); + if (first) { - fl_index += 1; - fl_size = fs_freelist_index_to_size (fl_index); - continue; + first->next = 0; + n_alloc = fs_freelist_index_to_size (fl_index); + goto done; } - return 0; } fl_index -= 1; @@ -400,100 +547,43 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, } } - fss->n_fl_chunk_bytes -= n_alloc; +done: + fss_fl_chunk_bytes_sub (fss, n_alloc); fsh_cached_bytes_sub (fsh, n_alloc); return first; } -static svm_fifo_t * -fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, - fifo_segment_slice_t * fss, - u32 data_bytes) +static int +fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, u32 batch_size) { - svm_fifo_chunk_t *c, *first = 0, *last = 0, *next; - u32 fl_index, fl_size, n_alloc = 0; - svm_fifo_t *f; - - f = fss->free_fifos; - if (!f) - { - if (PREDICT_FALSE (fsh_n_free_bytes (fsh) < sizeof (svm_fifo_t))) - return 0; + svm_fifo_shared_t *f, *head = 0, *tail; + uword size; + u8 *fmem; + int i; - void *oldheap = ssvm_push_heap (fsh->ssvm_sh); - f = clib_mem_alloc_aligned_or_null (sizeof (*f), CLIB_CACHE_LINE_BYTES); - ssvm_pop_heap (oldheap); - if (!f) - return 0; - memset (f, 0, sizeof (*f)); - fsh_free_bytes_sub (fsh, sizeof (*f)); - } - else - { - fss->free_fifos = f->next; - } + ASSERT (batch_size != 0); - fl_index = fs_freelist_for_size (data_bytes); - if (fl_index > 0) - fl_index -= 1; + size = (uword) sizeof (*f) * batch_size; - fl_size = fs_freelist_index_to_size (fl_index); + fmem = fsh_alloc_aligned (fsh, size, CLIB_CACHE_LINE_BYTES); + if (fmem == 0) + return -1; - while (data_bytes) + /* Carve fifo hdr space */ + tail = f = (svm_fifo_shared_t *) fmem; + for (i = 0; i < batch_size; i++) { - c = fss->free_chunks[fl_index]; - if (c) - { - fss->free_chunks[fl_index] = c->next; - if (!last) - last = c; - c->next = first; - first = c; - n_alloc += fl_size; - data_bytes -= clib_min (fl_size, data_bytes); - } - else - { - /* Failed to allocate with smaller chunks */ - if (fl_index == 0) - { - /* free all chunks if any allocated */ - c = first; - while (c) - { - fl_index = fs_freelist_for_size (c->length); - fl_size = fs_freelist_index_to_size (fl_index); - next = c->next; - c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; - fss->n_fl_chunk_bytes += fl_size; - n_alloc -= fl_size; - data_bytes += fl_size; - c = next; - } - first = last = 0; - fl_index = fs_freelist_for_size (data_bytes); - if (fss->free_chunks[fl_index + 1]) - { - fl_index += 1; - fl_size = fs_freelist_index_to_size (fl_index); - continue; - } - - f->next = fss->free_fifos; - fss->free_fifos = f; - return 0; - } - fl_index -= 1; - fl_size = fl_size >> 1; - } + clib_memset (f, 0, sizeof (*f)); + f->next = fs_sptr (fsh, head); + head = f; + fmem += sizeof (*f); + f = (svm_fifo_shared_t *) fmem; } - f->start_chunk = first; - f->end_chunk = last; - fss->n_fl_chunk_bytes -= n_alloc; - fsh_cached_bytes_sub (fsh, n_alloc); - return f; + fss_fifo_free_list_push_list (fsh, fss, head, tail); + + return 0; } static int @@ -501,43 +591,38 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 fl_index, u32 batch_size) { + svm_fifo_chunk_t *c, *head = 0, *tail; + uword size, total_chunk_bytes; u32 rounded_data_size; - svm_fifo_chunk_t *c; - void *oldheap; - uword size; u8 *cmem; int i; + ASSERT (batch_size != 0); + rounded_data_size = fs_freelist_index_to_size (fl_index); + total_chunk_bytes = (uword) batch_size *rounded_data_size; size = (uword) (sizeof (*c) + rounded_data_size) * batch_size; - oldheap = ssvm_push_heap (fsh->ssvm_sh); - cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, - 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - ssvm_pop_heap (oldheap); - - /* Out of space.. */ + cmem = fsh_alloc_aligned (fsh, size, 8 /* chunk hdr is 24B */); if (cmem == 0) return -1; /* Carve fifo + chunk space */ + tail = c = (svm_fifo_chunk_t *) cmem; for (i = 0; i < batch_size; i++) { - c = (svm_fifo_chunk_t *) cmem; c->start_byte = 0; c->length = rounded_data_size; - c->enq_rb_index = RBTREE_TNIL_INDEX; - c->deq_rb_index = RBTREE_TNIL_INDEX; - c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; + c->next = fs_chunk_sptr (fsh, head); + head = c; cmem += sizeof (*c) + rounded_data_size; + c = (svm_fifo_chunk_t *) cmem; } + fss_chunk_free_list_push_list (fsh, fss, fl_index, head, tail); fss->num_chunks[fl_index] += batch_size; - fss->n_fl_chunk_bytes += batch_size * rounded_data_size; - fsh_cached_bytes_add (fsh, batch_size * rounded_data_size); - fsh_free_bytes_sub (fsh, size); + fss_fl_chunk_bytes_add (fss, total_chunk_bytes); + fsh_cached_bytes_add (fsh, total_chunk_bytes); return 0; } @@ -547,224 +632,142 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 fl_index, u32 batch_size) { - u32 hdrs, rounded_data_size; - svm_fifo_chunk_t *c; - svm_fifo_t *f; - void *oldheap; - uword size; - u8 *fmem; - int i; - - rounded_data_size = fs_freelist_index_to_size (fl_index); - hdrs = sizeof (*f) + sizeof (*c); - size = (uword) (hdrs + rounded_data_size) * batch_size; - - oldheap = ssvm_push_heap (fsh->ssvm_sh); - fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, - 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - ssvm_pop_heap (oldheap); - - /* Out of space.. */ - if (fmem == 0) - return -1; - - /* Carve fifo hdr space */ - for (i = 0; i < batch_size; i++) - { - f = (svm_fifo_t *) fmem; - memset (f, 0, sizeof (*f)); - f->next = fss->free_fifos; - fss->free_fifos = f; - fmem += sizeof (*f); - } - - /* Carve chunk space */ - for (i = 0; i < batch_size; i++) - { - c = (svm_fifo_chunk_t *) fmem; - c->start_byte = 0; - c->length = rounded_data_size; - c->enq_rb_index = RBTREE_TNIL_INDEX; - c->deq_rb_index = RBTREE_TNIL_INDEX; - c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; - fmem += sizeof (svm_fifo_chunk_t) + rounded_data_size; - } - - fss->num_chunks[fl_index] += batch_size; - fss->n_fl_chunk_bytes += batch_size * rounded_data_size; - fsh_cached_bytes_add (fsh, batch_size * rounded_data_size); - fsh_free_bytes_sub (fsh, size); - - return 0; + if (fsh_try_alloc_fifo_hdr_batch (fsh, fss, batch_size)) + return 0; + return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size); } -/** - * Try to allocate new fifo - * - * Tries the following steps in order: - * - grab fifo and chunk from freelists - * - batch fifo and chunk allocation - * - single fifo allocation - * - grab multiple fifo chunks from freelists - */ -static svm_fifo_t * -fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, - u32 data_bytes) +static svm_fifo_shared_t * +fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - u32 fifo_sz, fl_index; - svm_fifo_t *f = 0; - uword n_free_bytes; - u32 min_size; - - min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096); - fl_index = fs_freelist_for_size (min_size); - - if (fl_index >= vec_len (fss->free_chunks)) - return 0; - - clib_spinlock_lock (&fss->chunk_lock); - - if (fss->free_fifos && fss->free_chunks[fl_index]) - { - f = fs_try_alloc_fifo_freelist (fss, fl_index); - if (f) - { - fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index)); - goto done; - } - } + svm_fifo_shared_t *sf; - fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t); - fifo_sz += 1 << max_log2 (min_size); - n_free_bytes = fsh_n_free_bytes (fsh); - - if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < n_free_bytes) - { - if (!fs_try_alloc_fifo_batch (fsh, fss, fl_index, - FIFO_SEGMENT_ALLOC_BATCH_SIZE)) - { - f = fs_try_alloc_fifo_freelist (fss, fl_index); - if (f) - { - fsh_cached_bytes_sub (fsh, - fs_freelist_index_to_size (fl_index)); - goto done; - } - } - else - { - fsh_check_mem (fsh); - n_free_bytes = fsh_n_free_bytes (fsh); - } - } - if (fifo_sz <= n_free_bytes) + if (!fss->free_fifos) { - void *oldheap = ssvm_push_heap (fsh->ssvm_sh); - f = svm_fifo_alloc (min_size); - ssvm_pop_heap (oldheap); - if (f) - { - clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1); - fsh_free_bytes_sub (fsh, fifo_sz); - goto done; - } - fsh_check_mem (fsh); + if (fsh_try_alloc_fifo_hdr_batch (fsh, fss, + FIFO_SEGMENT_ALLOC_BATCH_SIZE)) + return 0; } - /* All failed, try to allocate min of data bytes and fifo sz */ - fifo_sz = clib_min (fifo_sz, data_bytes); - if (fifo_sz <= fss->n_fl_chunk_bytes) - f = fs_try_alloc_fifo_freelist_multi_chunk (fsh, fss, fifo_sz); -done: - clib_spinlock_unlock (&fss->chunk_lock); + sf = fss_fifo_free_list_pop (fsh, fss); + clib_memset (sf, 0, sizeof (*sf)); - if (f) - { - f->size = data_bytes; - f->fs_hdr = fsh; - } - return f; + return sf; } -svm_fifo_chunk_t * -fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size) +static svm_fifo_chunk_t * +fsh_try_alloc_chunk (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, u32 data_bytes) { - fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; - int fl_index; - - fl_index = fs_freelist_for_size (chunk_size); - fss = fsh_slice_get (fsh, slice_index); - - clib_spinlock_lock (&fss->chunk_lock); + u32 fl_index; - ASSERT (vec_len (fss->free_chunks) > fl_index); - c = fss->free_chunks[fl_index]; + fl_index = fs_freelist_for_size (data_bytes); +free_list: + c = fss_chunk_free_list_pop (fsh, fss, fl_index); if (c) { - fss->free_chunks[fl_index] = c->next; c->next = 0; - fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); + fss_fl_chunk_bytes_sub (fss, fs_freelist_index_to_size (fl_index)); fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index)); } else { - void *oldheap; + u32 chunk_size, batch = FIFO_SEGMENT_ALLOC_BATCH_SIZE; uword n_free; - u32 batch; chunk_size = fs_freelist_index_to_size (fl_index); n_free = fsh_n_free_bytes (fsh); if (chunk_size <= n_free) { - oldheap = ssvm_push_heap (fsh->ssvm_sh); - c = svm_fifo_chunk_alloc (chunk_size); - ssvm_pop_heap (oldheap); - - if (c) - { - clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1); - fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c)); - goto done; - } - - fsh_check_mem (fsh); - n_free = fsh_n_free_bytes (fsh); + batch = chunk_size * batch <= n_free ? batch : 1; + if (!fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch)) + goto free_list; } - if (chunk_size <= fss->n_fl_chunk_bytes) + /* Failed to allocate larger chunk, try to allocate multi-chunk + * that is close to what was actually requested */ + if (data_bytes <= fss_fl_chunk_bytes (fss)) { - c = fs_try_alloc_multi_chunk (fsh, fss, chunk_size); + c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes); if (c) goto done; batch = n_free / FIFO_SEGMENT_MIN_FIFO_SIZE; if (!batch || fsh_try_alloc_chunk_batch (fsh, fss, 0, batch)) - { - fsh_check_mem (fsh); - goto done; - } + goto done; } - if (chunk_size <= fss->n_fl_chunk_bytes + n_free) + if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free) { u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE; - - batch = (chunk_size - fss->n_fl_chunk_bytes) / min_size; + if (n_free < min_size) + goto done; + batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size; batch = clib_min (batch + 1, n_free / min_size); if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch)) - { - fsh_check_mem (fsh); - goto done; - } - c = fs_try_alloc_multi_chunk (fsh, fss, chunk_size); + goto done; + c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes); } } done: - clib_spinlock_unlock (&fss->chunk_lock); + return c; +} + +/** + * Try to allocate new fifo + * + * Tries the following steps in order: + * - grab fifo and chunk from freelists + * - batch fifo and chunk allocation + * - single fifo allocation + * - grab multiple fifo chunks from freelists + */ +static svm_fifo_shared_t * +fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes) +{ + fifo_segment_slice_t *fss; + u32 fl_index, min_size; + svm_fifo_chunk_t *c; + svm_fifo_shared_t *sf = 0; + + fss = fsh_slice_get (fsh, slice_index); + min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096); + fl_index = fs_freelist_for_size (min_size); + + if (!fss_chunk_fl_index_is_valid (fss, fl_index)) + return 0; + + sf = fsh_try_alloc_fifo_hdr (fsh, fss); + if (!sf) + return 0; + + c = fsh_try_alloc_chunk (fsh, fss, min_size); + if (!c) + { + fss_fifo_free_list_push (fsh, fss, sf); + return 0; + } + + sf->start_chunk = fs_chunk_sptr (fsh, c); + while (c->next) + c = fs_chunk_ptr (fsh, c->next); + sf->end_chunk = fs_chunk_sptr (fsh, c); + sf->size = data_bytes; + sf->slice_index = slice_index; + + return sf; +} + +svm_fifo_chunk_t * +fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size) +{ + fifo_segment_slice_t *fss; + svm_fifo_chunk_t *c; + + fss = fsh_slice_get (fsh, slice_index); + c = fsh_try_alloc_chunk (fsh, fss, chunk_size); return c; } @@ -773,29 +776,21 @@ static void fsh_slice_collect_chunks (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, svm_fifo_chunk_t * c) { + u32 n_collect = 0, fl_index; svm_fifo_chunk_t *next; - int fl_index; - u32 n_collect = 0; - - clib_spinlock_lock (&fss->chunk_lock); while (c) { - CLIB_MEM_UNPOISON (c, sizeof (*c)); - next = c->next; + clib_mem_unpoison (c, sizeof (*c)); + next = fs_chunk_ptr (fsh, c->next); fl_index = fs_freelist_for_size (c->length); - c->next = fss->free_chunks[fl_index]; - c->enq_rb_index = RBTREE_TNIL_INDEX; - c->deq_rb_index = RBTREE_TNIL_INDEX; - fss->free_chunks[fl_index] = c; + fss_chunk_free_list_push (fsh, fss, fl_index, c); n_collect += fs_freelist_index_to_size (fl_index); c = next; } - fss->n_fl_chunk_bytes += n_collect; + fss_fl_chunk_bytes_add (fss, n_collect); fsh_cached_bytes_add (fsh, n_collect); - - clib_spinlock_unlock (&fss->chunk_lock); } void @@ -807,29 +802,44 @@ fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index, fsh_slice_collect_chunks (fsh, fss, c); } -static inline void -fss_fifo_add_active_list (fifo_segment_slice_t * fss, svm_fifo_t * f) +svm_fifo_t * +fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index) { - if (fss->fifos) - { - fss->fifos->prev = f; - f->next = fss->fifos; - } - fss->fifos = f; + fifo_slice_private_t *pfss = &fs->slices[slice_index]; + svm_fifo_t *f; + + f = clib_mem_bulk_alloc (pfss->fifos); + clib_memset (f, 0, sizeof (*f)); + return f; } -static inline void -fss_fifo_del_active_list (fifo_segment_slice_t * fss, svm_fifo_t * f) +void +fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f, u32 slice_index) { - if (f->flags & SVM_FIFO_F_LL_TRACKED) - { - if (f->prev) - f->prev->next = f->next; - else - fss->fifos = f->next; - if (f->next) - f->next->prev = f->prev; - } + fifo_slice_private_t *pfss; + + if (CLIB_DEBUG) + clib_memset (f, 0xfc, sizeof (*f)); + + pfss = &fs->slices[slice_index]; + clib_mem_bulk_free (pfss->fifos, f); +} + +void +fifo_segment_cleanup (fifo_segment_t *fs) +{ + int slice_index; + svm_msg_q_t *mq = 0; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + clib_mem_bulk_destroy (fs->slices[slice_index].fifos); + + vec_free (fs->slices); + + vec_foreach (mq, fs->mqs) + svm_msg_q_cleanup (mq); + + vec_free (fs->mqs); } /** @@ -840,30 +850,39 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype) { fifo_segment_header_t *fsh = fs->h; + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; svm_fifo_t *f = 0; ASSERT (slice_index < fs->n_slices); - if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_chunk_size)) + if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size)) return 0; - fss = fsh_slice_get (fsh, slice_index); - f = fs_try_alloc_fifo (fsh, fss, data_bytes); - if (!f) + sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes); + if (!sf) goto done; - f->slice_index = slice_index; + f = fs_fifo_alloc (fs, slice_index); + f->fs_hdr = fsh; + f->shr = sf; svm_fifo_init (f, data_bytes); + f->segment_manager = fs->sm_index; + f->segment_index = fs->fs_index; + + fss = fsh_slice_get (fsh, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track * only one. */ if (ftype == FIFO_SEGMENT_RX_FIFO) { - fss_fifo_add_active_list (fss, f); + pfss_fifo_add_active_list (pfss, f); f->flags |= SVM_FIFO_F_LL_TRACKED; } @@ -874,6 +893,30 @@ done: return (f); } +svm_fifo_t * +fifo_segment_alloc_fifo_w_offset (fifo_segment_t *fs, uword offset) +{ + svm_fifo_t *f = fs_fifo_alloc (fs, 0); + svm_fifo_shared_t *sf; + + sf = (svm_fifo_shared_t *) ((u8 *) fs->h + offset); + f->fs_hdr = fs->h; + f->shr = sf; + + f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->segment_index = SVM_FIFO_INVALID_INDEX; + f->refcnt = 1; + return f; +} + +svm_fifo_t * +fifo_segment_duplicate_fifo (fifo_segment_t *fs, svm_fifo_t *f) +{ + svm_fifo_t *nf = fs_fifo_alloc (fs, 0); + clib_memcpy (nf, f, sizeof (*f)); + return nf; +} + /** * Free fifo allocated in fifo segment */ @@ -881,132 +924,270 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh = fs->h; + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; ASSERT (f->refcnt > 0); if (--f->refcnt > 0) return; - fss = fsh_slice_get (fsh, f->slice_index); + /* + * Cleanup shared state + */ + + sf = f->shr; + fss = fsh_slice_get (fsh, sf->slice_index); + pfss = fs_slice_private_get (fs, sf->slice_index); + + /* Free fifo chunks */ + fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk)); + + sf->start_chunk = sf->end_chunk = 0; + sf->head_chunk = sf->tail_chunk = 0; + + /* Add to free list */ + fss_fifo_free_list_push (fsh, fss, sf); + + fss->virtual_mem -= svm_fifo_size (f); + + /* + * Cleanup private state + */ /* Remove from active list. Only rx fifos are tracked */ if (f->flags & SVM_FIFO_F_LL_TRACKED) { - fss_fifo_del_active_list (fss, f); + pfss_fifo_del_active_list (pfss, f); f->flags &= ~SVM_FIFO_F_LL_TRACKED; } - /* Free fifo chunks */ - fsh_slice_collect_chunks (fsh, fss, f->start_chunk); - - f->start_chunk = f->end_chunk = 0; - f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; - - /* not allocated on segment heap */ svm_fifo_free_chunk_lookup (f); svm_fifo_free_ooo_data (f); if (CLIB_DEBUG) { - f->master_session_index = ~0; + sf->master_session_index = ~0; f->master_thread_index = ~0; } - fss->virtual_mem -= svm_fifo_size (f); - - /* Add to free list */ - f->next = fss->free_fifos; + f->ooo_enq = f->ooo_deq = 0; f->prev = 0; - fss->free_fifos = f; + + fs_fifo_free (fs, f, f->shr->slice_index); fsh_active_fifos_update (fsh, -1); } void -fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f) +fifo_segment_free_client_fifo (fifo_segment_t *fs, svm_fifo_t *f) +{ + fs_fifo_free (fs, f, 0 /* clients attach fifos in slice 0 */); +} + +void +fifo_segment_detach_fifo (fifo_segment_t *fs, svm_fifo_t **f) { + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - u32 fl_index; + svm_fifo_t *of = *f; + u32 slice_index; - ASSERT (f->refcnt == 1); + slice_index = of->master_thread_index; + fss = fsh_slice_get (fs->h, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + fss->virtual_mem -= svm_fifo_size (of); + if (of->flags & SVM_FIFO_F_LL_TRACKED) + pfss_fifo_del_active_list (pfss, of); - fss = fsh_slice_get (fs->h, f->slice_index); - fss->virtual_mem -= svm_fifo_size (f); - if (f->flags & SVM_FIFO_F_LL_TRACKED) - fss_fifo_del_active_list (fss, f); + /* Collect chunks that were provided in return for those detached */ + fsh_slice_collect_chunks (fs->h, fss, of->chunks_at_attach); + of->chunks_at_attach = 0; - c = f->start_chunk; - while (c) - { - fl_index = fs_freelist_for_size (c->length); - clib_atomic_fetch_sub_rel (&fss->num_chunks[fl_index], 1); - c = c->next; - } + /* Collect hdr that was provided in return for the detached */ + fss_fifo_free_list_push (fs->h, fss, of->hdr_at_attach); + of->hdr_at_attach = 0; + + clib_mem_bulk_free (pfss->fifos, *f); + *f = 0; } void -fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f, - u32 slice_index) +fifo_segment_attach_fifo (fifo_segment_t *fs, svm_fifo_t **f, u32 slice_index) { + svm_fifo_chunk_t *c, *nc, *pc = 0; + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - u32 fl_index; + svm_fifo_t *nf, *of; - f->slice_index = slice_index; - fss = fsh_slice_get (fs->h, f->slice_index); - fss->virtual_mem += svm_fifo_size (f); - if (f->flags & SVM_FIFO_F_LL_TRACKED) - fss_fifo_add_active_list (fss, f); + nf = fs_fifo_alloc (fs, slice_index); + clib_memcpy_fast (nf, *f, sizeof (*nf)); - c = f->start_chunk; - while (c) + fss = fsh_slice_get (fs->h, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + fss->virtual_mem += svm_fifo_size (nf); + nf->next = nf->prev = 0; + if (nf->flags & SVM_FIFO_F_LL_TRACKED) + pfss_fifo_add_active_list (pfss, nf); + + /* Allocate shared hdr and chunks to be collected at detach in return + * for those that are being attached now */ + of = *f; + of->hdr_at_attach = fsh_try_alloc_fifo_hdr (fs->h, fss); + + c = fs_chunk_ptr (fs->h, nf->shr->start_chunk); + of->chunks_at_attach = pc = fsh_try_alloc_chunk (fs->h, fss, c->length); + + while ((c = fs_chunk_ptr (fs->h, c->next))) { - fl_index = fs_freelist_for_size (c->length); - clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1); - c = c->next; + nc = fsh_try_alloc_chunk (fs->h, fss, c->length); + pc->next = fs_chunk_sptr (fs->h, nc); + pc = nc; } + + nf->shr->slice_index = slice_index; + *f = nf; } -int -fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, - u32 batch_size) +uword +fifo_segment_fifo_offset (svm_fifo_t *f) +{ + return (u8 *) f->shr - (u8 *) f->fs_hdr; +} + +svm_fifo_chunk_t * +fifo_segment_alloc_chunk_w_slice (fifo_segment_t *fs, u32 slice_index, + u32 chunk_size) { fifo_segment_header_t *fsh = fs->h; fifo_segment_slice_t *fss; - svm_fifo_t *f; - void *oldheap; - uword size; - u8 *fmem; - int i; fss = fsh_slice_get (fsh, slice_index); - size = (uword) (sizeof (*f)) * batch_size; + return fsh_try_alloc_chunk (fsh, fss, chunk_size); +} - oldheap = ssvm_push_heap (fsh->ssvm_sh); - fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, - 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - ssvm_pop_heap (oldheap); +void +fifo_segment_collect_chunk (fifo_segment_t *fs, u32 slice_index, + svm_fifo_chunk_t *c) +{ + fsh_collect_chunks (fs->h, slice_index, c); +} - /* Out of space.. */ - if (fmem == 0) - return -1; +uword +fifo_segment_chunk_offset (fifo_segment_t *fs, svm_fifo_chunk_t *c) +{ + return (u8 *) c - (u8 *) fs->h; +} - /* Carve fifo + chunk space */ - for (i = 0; i < batch_size; i++) +svm_msg_q_t * +fifo_segment_msg_q_alloc (fifo_segment_t *fs, u32 mq_index, + svm_msg_q_cfg_t *cfg) +{ + fifo_segment_header_t *fsh = fs->h; + svm_msg_q_shared_t *smq; + svm_msg_q_t *mq; + void *base; + u32 size; + + if (!fs->mqs) { - f = (svm_fifo_t *) fmem; - memset (f, 0, sizeof (*f)); - f->next = fss->free_fifos; - fss->free_fifos = f; - fmem += sizeof (*f); + u32 n_mqs = clib_max (fs->h->n_mqs, 1); + vec_validate (fs->mqs, n_mqs - 1); } - fsh_free_bytes_sub (fsh, size); + size = svm_msg_q_size_to_alloc (cfg); + base = fsh_alloc_aligned (fsh, size, 8); + if (!base) + return 0; + + fsh->n_reserved_bytes += size; - return 0; + smq = svm_msg_q_init (base, cfg); + mq = vec_elt_at_index (fs->mqs, mq_index); + svm_msg_q_attach (mq, smq); + + return mq; +} + +svm_msg_q_t * +fifo_segment_msg_q_attach (fifo_segment_t *fs, uword offset, u32 mq_index) +{ + svm_msg_q_t *mq; + + if (!fs->mqs) + { + u32 n_mqs = clib_max (fs->h->n_mqs, 1); + vec_validate (fs->mqs, n_mqs - 1); + } + + mq = vec_elt_at_index (fs->mqs, mq_index); + + if (!mq->q.shr) + { + svm_msg_q_shared_t *smq; + smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset); + svm_msg_q_attach (mq, smq); + } + + ASSERT (fifo_segment_msg_q_offset (fs, mq_index) == offset); + + return mq; +} + +void +fifo_segment_msg_qs_discover (fifo_segment_t *fs, int *fds, u32 n_fds) +{ + svm_msg_q_shared_t *smq; + u32 n_mqs, size, i; + uword offset = 0, n_alloced; + svm_msg_q_t *mq; + + n_mqs = fs->h->n_mqs; + if (n_fds && n_mqs != n_fds) + { + clib_warning ("expected %u fds got %u", n_mqs, n_fds); + return; + } + + vec_validate (fs->mqs, n_mqs - 1); + n_alloced = fs->h->n_reserved_bytes - fs->h->start_byte_index; + ASSERT (n_alloced % n_mqs == 0); + size = n_alloced / n_mqs; + + offset = fs->h->start_byte_index; + for (i = 0; i < n_mqs; i++) + { + mq = vec_elt_at_index (fs->mqs, i); + smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset); + svm_msg_q_attach (mq, smq); + if (n_fds) + svm_msg_q_set_eventfd (mq, fds[i]); + offset += size; + } +} + +uword +fifo_segment_msg_q_offset (fifo_segment_t *fs, u32 mq_index) +{ + svm_msg_q_t *mq = vec_elt_at_index (fs->mqs, mq_index); + + if (mq->q.shr == 0) + return ~0ULL; + + return (uword) ((u8 *) mq->q.shr - (u8 *) fs->h) - + sizeof (svm_msg_q_shared_t); +} + +int +fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, + u32 batch_size) +{ + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + + fss = fsh_slice_get (fsh, slice_index); + return fsh_try_alloc_fifo_hdr_batch (fsh, fss, batch_size); } int @@ -1014,13 +1195,8 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, u32 chunk_size, u32 batch_size) { fifo_segment_header_t *fsh = fs->h; - u32 rounded_data_size, fl_index; fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - void *oldheap; - uword size; - u8 *cmem; - int i; + u32 fl_index; if (!fs_chunk_size_is_valid (fsh, chunk_size)) { @@ -1029,38 +1205,9 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, } fl_index = fs_freelist_for_size (chunk_size); - rounded_data_size = fs_freelist_index_to_size (fl_index); - size = (uword) (sizeof (*c) + rounded_data_size) * batch_size; - - oldheap = ssvm_push_heap (fsh->ssvm_sh); - cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, - 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - ssvm_pop_heap (oldheap); - - /* Out of space.. */ - if (cmem == 0) - return -1; - fss = fsh_slice_get (fsh, slice_index); - /* Carve fifo + chunk space */ - for (i = 0; i < batch_size; i++) - { - c = (svm_fifo_chunk_t *) cmem; - c->start_byte = 0; - c->length = rounded_data_size; - c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; - cmem += sizeof (*c) + rounded_data_size; - fsh_cached_bytes_add (fsh, rounded_data_size); - } - - fss->num_chunks[fl_index] += batch_size; - fss->n_fl_chunk_bytes += batch_size * rounded_data_size; - fsh_free_bytes_sub (fsh, size); - - return 0; + return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size); } /** @@ -1103,7 +1250,7 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, /* Calculate space requirements */ pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size; - space_available = fsh_free_space (fsh); + space_available = fsh_n_free_bytes (fsh); pairs_to_alloc = space_available / pair_size; pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs); pairs_per_slice = pairs_to_alloc / fs->n_slices; @@ -1114,8 +1261,11 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, for (i = 0; i < fs->n_slices; i++) { - fss = fsh_slice_get (fsh, i); alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs); + if (0 == alloc_now) + break; + + fss = fsh_slice_get (fsh, i); if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now)) clib_warning ("rx prealloc failed: pairs %u", alloc_now); if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now)) @@ -1136,18 +1286,18 @@ fifo_segment_num_fifos (fifo_segment_t * fs) } static u32 -fs_slice_num_free_fifos (fifo_segment_slice_t * fss) +fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - svm_fifo_t *f; + svm_fifo_shared_t *f; u32 count = 0; - f = fss->free_fifos; + f = fs_ptr (fsh, fss->free_fifos); if (f == 0) return 0; while (f) { - f = f->next; + f = fs_ptr (fsh, f->next); count++; } return count; @@ -1164,13 +1314,14 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - count += fs_slice_num_free_fifos (fss); + count += fs_slice_num_free_fifos (fsh, fss); } return count; } static u32 -fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) +fs_slice_num_free_chunks (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 size) { u32 count = 0, rounded_size, fl_index; svm_fifo_chunk_t *c; @@ -1179,15 +1330,15 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) /* Count all free chunks? */ if (size == ~0) { - for (i = 0; i < vec_len (fss->free_chunks); i++) + for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { - c = fss->free_chunks[i]; + c = fss_chunk_free_list_head (fsh, fss, i); if (c == 0) continue; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } } @@ -1197,16 +1348,16 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) rounded_size = (1 << (max_log2 (size))); fl_index = fs_freelist_for_size (rounded_size); - if (fl_index >= vec_len (fss->free_chunks)) + if (fl_index >= FS_CHUNK_VEC_LEN) return 0; - c = fss->free_chunks[fl_index]; + c = fss_chunk_free_list_head (fsh, fss, fl_index); if (c == 0) return 0; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } return count; @@ -1223,33 +1374,37 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - count += fs_slice_num_free_chunks (fss, size); + count += fs_slice_num_free_chunks (fsh, fss, size); } return count; } -void -fifo_segment_update_free_bytes (fifo_segment_t * fs) +uword +fifo_segment_size (fifo_segment_t * fs) { - fsh_update_free_bytes (fs->h); + return fs->h->max_byte_index - fs->h->n_reserved_bytes; } -uword -fifo_segment_size (fifo_segment_t * fs) +static u8 +fs_has_reached_mem_limit (fifo_segment_t *fs) { - return fs->ssvm.ssvm_size; + return (fs->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0; } -u8 -fsh_has_reached_mem_limit (fifo_segment_header_t * fsh) +static void +fs_reset_mem_limit (fifo_segment_t *fs) { - return (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0; + fs->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT; } -void -fsh_reset_mem_limit (fifo_segment_header_t * fsh) +void * +fifo_segment_alloc (fifo_segment_t *fs, uword size) { - fsh->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT; + void *rv = fsh_alloc (fs->h, size); + /* Mark externally allocated bytes as reserved. This helps + * @ref fifo_segment_size report bytes used only for fifos */ + fs->h->n_reserved_bytes += size; + return rv; } uword @@ -1281,7 +1436,7 @@ fifo_segment_fl_chunk_bytes (fifo_segment_t * fs) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - n_bytes += fss->n_fl_chunk_bytes; + n_bytes += fss_fl_chunk_bytes (fss); } return n_bytes; @@ -1296,11 +1451,10 @@ fifo_segment_has_fifos (fifo_segment_t * fs) svm_fifo_t * fifo_segment_get_slice_fifo_list (fifo_segment_t * fs, u32 slice_index) { - fifo_segment_header_t *fsh = fs->h; - fifo_segment_slice_t *fss; + fifo_slice_private_t *pfss; - fss = fsh_slice_get (fsh, slice_index); - return fss->fifos; + pfss = fs_slice_private_get (fs, slice_index); + return pfss->active_fifos; } u8 @@ -1315,26 +1469,26 @@ fifo_segment_get_mem_usage (fifo_segment_t * fs) } fifo_segment_mem_status_t -fifo_segment_determine_status (fifo_segment_header_t * fsh, u8 usage) +fifo_segment_determine_status (fifo_segment_t *fs, u8 usage) { - if (!fsh->high_watermark || !fsh->low_watermark) + if (!fs->high_watermark || !fs->low_watermark) return MEMORY_PRESSURE_NO_PRESSURE; /* once the no-memory is detected, the status continues * until memory usage gets below the high watermark */ - if (fsh_has_reached_mem_limit (fsh)) + if (fs_has_reached_mem_limit (fs)) { - if (usage >= fsh->high_watermark) + if (usage >= fs->high_watermark) return MEMORY_PRESSURE_NO_MEMORY; else - fsh_reset_mem_limit (fsh); + fs_reset_mem_limit (fs); } - if (usage >= fsh->high_watermark) + if (usage >= fs->high_watermark) return MEMORY_PRESSURE_HIGH_PRESSURE; - else if (usage >= fsh->low_watermark) + else if (usage >= fs->low_watermark) return MEMORY_PRESSURE_LOW_PRESSURE; return MEMORY_PRESSURE_NO_PRESSURE; @@ -1343,10 +1497,9 @@ fifo_segment_determine_status (fifo_segment_header_t * fsh, u8 usage) fifo_segment_mem_status_t fifo_segment_get_mem_status (fifo_segment_t * fs) { - fifo_segment_header_t *fsh = fs->h; u8 usage = fifo_segment_get_mem_usage (fs); - return fifo_segment_determine_status (fsh, usage); + return fifo_segment_determine_status (fs, usage); } u8 * @@ -1391,22 +1544,15 @@ format_fifo_segment (u8 * s, va_list * args) f64 usage; fifo_segment_mem_status_t mem_st; - indent = format_get_indent (s) + 2; - - if (fs == 0) - { - s = format (s, "%-20s%10s%15s%15s%15s%15s", "Name", "Type", - "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address"); - return s; - } + indent = format_get_indent (s); fifo_segment_info (fs, &address, &size); active_fifos = fifo_segment_num_fifos (fs); free_fifos = fifo_segment_num_free_fifos (fs); - s = format (s, "%-20v%10U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm), - format_fifo_segment_type, fs, size >> 20ULL, active_fifos, - free_fifos, address); + s = format (s, "%U%v type: %U size: %U active fifos: %u", format_white_space, + 2, ssvm_name (&fs->ssvm), format_fifo_segment_type, fs, + format_memory_size, size, active_fifos); if (!verbose) return s; @@ -1415,24 +1561,23 @@ format_fifo_segment (u8 * s, va_list * args) free_chunks = fifo_segment_num_free_chunks (fs, ~0); if (free_chunks) - s = - format (s, "\n\n%UFree/Allocated chunks by size:\n", format_white_space, - indent + 2); + s = format (s, "\n\n%UFree/Allocated chunks by size:\n", + format_white_space, indent + 2); else s = format (s, "\n"); for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - for (i = 0; i < vec_len (fss->free_chunks); i++) + for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { - c = fss->free_chunks[i]; + c = fss_chunk_free_list_head (fsh, fss, i); if (c == 0 && fss->num_chunks[i] == 0) continue; count = 0; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } @@ -1447,14 +1592,13 @@ format_fifo_segment (u8 * s, va_list * args) fifo_hdr = free_fifos * sizeof (svm_fifo_t); est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs); est_free_seg_bytes = fifo_segment_free_bytes (fs); - fifo_segment_update_free_bytes (fs); free_seg_bytes = fifo_segment_free_bytes (fs); tracked_cached_bytes = fifo_segment_cached_bytes (fs); allocated = fifo_segment_size (fs); in_use = fifo_segment_size (fs) - est_free_seg_bytes - tracked_cached_bytes; usage = (100.0 * in_use) / allocated; mem_st = fifo_segment_get_mem_status (fs); - virt = fsh_virtual_mem (fsh); + virt = fs_virtual_mem (fs); reserved = fsh->n_reserved_bytes; s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu) reserved:" @@ -1467,8 +1611,8 @@ format_fifo_segment (u8 * s, va_list * args) format_memory_size, chunk_bytes, chunk_bytes, format_memory_size, est_chunk_bytes, est_chunk_bytes, format_memory_size, tracked_cached_bytes, tracked_cached_bytes); - s = format (s, "%Ufifo active: %u hdr free bytes: %U (%u) \n", - format_white_space, indent + 2, fsh->n_active_fifos, + s = format (s, "%Ufifo active: %u hdr free: %u bytes: %U (%u) \n", + format_white_space, indent + 2, fsh->n_active_fifos, free_fifos, format_memory_size, fifo_hdr, fifo_hdr); s = format (s, "%Usegment usage: %.2f%% (%U / %U) virt: %U status: %s\n", format_white_space, indent + 2, usage, format_memory_size,