X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Ffifo_segment.c;h=986778dfe82a31e08ffdbc46ad6af3258dd3fc57;hb=83a09c6b3cc2444132f90826b7e2ceb2a3f3347c;hp=b4c3cbe6c6ded0179ef7c99bbcdca267f616b4ac;hpb=4b8011e72ea22d17942f2e57a34a4fa3a3d81af5;p=vpp.git diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index b4c3cbe6c6d..986778dfe82 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -14,6 +14,7 @@ */ #include +#include static inline void * fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align) @@ -51,6 +52,13 @@ fsh_slice_get (fifo_segment_header_t * fsh, u32 slice_index) return &fsh->slices[slice_index]; } +static inline fifo_slice_private_t * +fs_slice_private_get (fifo_segment_t *fs, u32 slice_index) +{ + ASSERT (slice_index < fs->n_slices); + return &fs->slices[slice_index]; +} + static char *fifo_segment_mem_status_strings[] = { #define _(sym,str) str, foreach_segment_mem_status @@ -147,28 +155,30 @@ fss_chunk_fl_index_is_valid (fifo_segment_slice_t * fss, u32 fl_index) } static void -fss_chunk_free_list_push (fifo_segment_slice_t * fss, u32 fl_index, - svm_fifo_chunk_t * c) +fss_chunk_free_list_push (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 fl_index, + svm_fifo_chunk_t *c) { fss_chunk_freelist_lock (fss); c->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = c; + fss->free_chunks[fl_index] = fs_chunk_sptr (fsh, c); fss_chunk_freelist_unlock (fss); } static void -fss_chunk_free_list_push_list (fifo_segment_slice_t * fss, u32 fl_index, - svm_fifo_chunk_t * head, - svm_fifo_chunk_t * tail) +fss_chunk_free_list_push_list (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 fl_index, + svm_fifo_chunk_t *head, svm_fifo_chunk_t *tail) { fss_chunk_freelist_lock (fss); tail->next = fss->free_chunks[fl_index]; - fss->free_chunks[fl_index] = head; + fss->free_chunks[fl_index] = fs_chunk_sptr (fsh, head); fss_chunk_freelist_unlock (fss); } static svm_fifo_chunk_t * -fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index) +fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, + u32 fl_index) { svm_fifo_chunk_t *c; @@ -182,7 +192,7 @@ fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index) return 0; } - c = fss->free_chunks[fl_index]; + c = fs_chunk_ptr (fsh, fss->free_chunks[fl_index]); fss->free_chunks[fl_index] = c->next; fss_chunk_freelist_unlock (fss); @@ -190,26 +200,52 @@ fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index) return c; } +static void +fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, + svm_fifo_shared_t *sf) +{ + sf->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, sf); +} + +static void +fss_fifo_free_list_push_list (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, + svm_fifo_shared_t *head, svm_fifo_shared_t *tail) +{ + tail->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, head); +} + +svm_fifo_shared_t * +fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) +{ + svm_fifo_shared_t *sf; + sf = fs_ptr (fsh, fss->free_fifos); + fss->free_fifos = sf->next; + return sf; +} + static inline void -fss_fifo_add_active_list (fifo_segment_slice_t * fss, svm_fifo_t * f) +pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f) { - if (fss->fifos) + if (pfss->active_fifos) { - fss->fifos->prev = f; - f->next = fss->fifos; + pfss->active_fifos->prev = f; + f->next = pfss->active_fifos; } - fss->fifos = f; + pfss->active_fifos = f; } static inline void -fss_fifo_del_active_list (fifo_segment_slice_t * fss, svm_fifo_t * f) +pfss_fifo_del_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f) { if (f->flags & SVM_FIFO_F_LL_TRACKED) { if (f->prev) f->prev->next = f->next; else - fss->fifos = f->next; + pfss->active_fifos = f->next; if (f->next) f->next->prev = f->prev; } @@ -239,7 +275,7 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size) int fifo_segment_init (fifo_segment_t * fs) { - u32 align = 8, offset = 2 * 4096, slices_sz; + u32 align = 8, offset = 2 * 4096, slices_sz, i; uword max_fifo, seg_start, seg_sz; fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; @@ -256,6 +292,7 @@ fifo_segment_init (fifo_segment_t * fs) seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align); fsh = uword_to_pointer (seg_start, void *); + CLIB_MEM_UNPOISON (fsh, seg_sz); memset (fsh, 0, sizeof (*fsh) + slices_sz); fsh->byte_index = sizeof (*fsh) + slices_sz; @@ -265,10 +302,20 @@ fifo_segment_init (fifo_segment_t * fs) fsh->max_log2_fifo_size = min_log2 (max_fifo); fsh->n_cached_bytes = 0; fsh->n_reserved_bytes = fsh->byte_index; + fsh->start_byte_index = fsh->byte_index; ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset); fs->max_byte_index = fsh->max_byte_index; - fs->h = sh->opaque[0] = fsh; + fs->h = fsh; + sh->opaque[0] = (void *) ((u8 *) fsh - (u8 *) fs->ssvm.sh); + + /* Allow random offsets */ + fs->ssvm.sh->ssvm_va = 0; + + vec_validate (fs->slices, fs->n_slices - 1); + for (i = 0; i < fs->n_slices; i++) + fs->slices[i].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); sh->ready = 1; return (0); @@ -323,7 +370,7 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) fs->ssvm.ssvm_size = a->segment_size; fs->ssvm.my_pid = getpid (); fs->ssvm.name = format (0, "%s%c", a->segment_name, 0); - fs->ssvm.requested_va = sm->next_baseva; + fs->ssvm.requested_va = 0; if (a->segment_type == SSVM_SEGMENT_MEMFD) fs->ssvm.fd = a->memfd_fd; else @@ -331,18 +378,19 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) if ((rv = ssvm_client_init (&fs->ssvm, a->segment_type))) { - _vec_len (fs) = vec_len (fs) - 1; + pool_put (sm->segments, fs); return (rv); } - /* Fish the segment header */ - fsh = fs->h = fs->ssvm.sh->opaque[0]; - /* Probably a segment without fifos */ - if (!fsh) + if (!fs->ssvm.sh->opaque[0]) goto done; + fsh = fs->h = (void *) fs->ssvm.sh + (uword) fs->ssvm.sh->opaque[0]; fs->max_byte_index = fsh->max_byte_index; + vec_validate (fs->slices, 0); + fs->slices[0].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); done: vec_add1 (a->new_segment_indices, fs - sm->segments); @@ -352,6 +400,7 @@ done: void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s) { + fifo_segment_cleanup (s); ssvm_delete (&s->ssvm); clib_memset (s, 0xfe, sizeof (*s)); pool_put (sm->segments, s); @@ -369,6 +418,14 @@ fifo_segment_get_segment (fifo_segment_main_t * sm, u32 segment_index) return pool_elt_at_index (sm->segments, segment_index); } +fifo_segment_t * +fifo_segment_get_segment_if_valid (fifo_segment_main_t *sm, u32 segment_index) +{ + if (pool_is_free_index (sm->segments, segment_index)) + return 0; + return pool_elt_at_index (sm->segments, segment_index); +} + void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size) { @@ -425,10 +482,10 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, while (req_bytes) { - c = fss_chunk_free_list_pop (fss, fl_index); + c = fss_chunk_free_list_pop (fsh, fss, fl_index); if (c) { - c->next = first; + c->next = fs_chunk_sptr (fsh, first); first = c; n_alloc += fl_size; req_bytes -= clib_min (fl_size, req_bytes); @@ -443,8 +500,8 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, while (c) { fl_index = fs_freelist_for_size (c->length); - next = c->next; - fss_chunk_free_list_push (fss, fl_index, c); + next = fs_chunk_ptr (fsh, c->next); + fss_chunk_free_list_push (fsh, fss, fl_index, c); c = next; } n_alloc = 0; @@ -454,7 +511,7 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh, fl_index = fs_freelist_for_size (data_bytes) + 1; if (!fss_chunk_fl_index_is_valid (fss, fl_index)) return 0; - first = fss_chunk_free_list_pop (fss, fl_index); + first = fss_chunk_free_list_pop (fsh, fss, fl_index); if (first) { first->next = 0; @@ -478,7 +535,7 @@ static int fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 batch_size) { - svm_fifo_t *f; + svm_fifo_shared_t *f, *head = 0, *tail; uword size; u8 *fmem; int i; @@ -492,15 +549,18 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, return -1; /* Carve fifo hdr space */ + tail = f = (svm_fifo_shared_t *) fmem; for (i = 0; i < batch_size; i++) { - f = (svm_fifo_t *) fmem; - memset (f, 0, sizeof (*f)); - f->next = fss->free_fifos; - fss->free_fifos = f; + clib_memset (f, 0, sizeof (*f)); + f->next = fs_sptr (fsh, head); + head = f; fmem += sizeof (*f); + f = (svm_fifo_shared_t *) fmem; } + fss_fifo_free_list_push_list (fsh, fss, head, tail); + return 0; } @@ -531,13 +591,13 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh, { c->start_byte = 0; c->length = rounded_data_size; - c->next = head; + c->next = fs_chunk_sptr (fsh, head); head = c; cmem += sizeof (*c) + rounded_data_size; c = (svm_fifo_chunk_t *) cmem; } - fss_chunk_free_list_push_list (fss, fl_index, head, tail); + fss_chunk_free_list_push_list (fsh, fss, fl_index, head, tail); fss->num_chunks[fl_index] += batch_size; fss_fl_chunk_bytes_add (fss, total_chunk_bytes); fsh_cached_bytes_add (fsh, total_chunk_bytes); @@ -555,11 +615,10 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size); } -static svm_fifo_t * -fsh_try_alloc_fifo_hdr (fifo_segment_header_t * fsh, - fifo_segment_slice_t * fss) +static svm_fifo_shared_t * +fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - svm_fifo_t *f; + svm_fifo_shared_t *sf; if (!fss->free_fifos) { @@ -568,10 +627,10 @@ fsh_try_alloc_fifo_hdr (fifo_segment_header_t * fsh, return 0; } - f = fss->free_fifos; - fss->free_fifos = f->next; - memset (f, 0, sizeof (*f)); - return f; + sf = fss_fifo_free_list_pop (fsh, fss); + clib_memset (sf, 0, sizeof (*sf)); + + return sf; } static svm_fifo_chunk_t * @@ -584,7 +643,7 @@ fsh_try_alloc_chunk (fifo_segment_header_t * fsh, fl_index = fs_freelist_for_size (data_bytes); free_list: - c = fss_chunk_free_list_pop (fss, fl_index); + c = fss_chunk_free_list_pop (fsh, fss, fl_index); if (c) { c->next = 0; @@ -642,40 +701,40 @@ done: * - single fifo allocation * - grab multiple fifo chunks from freelists */ -static svm_fifo_t * -fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, - u32 data_bytes) +static svm_fifo_shared_t * +fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes) { + fifo_segment_slice_t *fss; u32 fl_index, min_size; svm_fifo_chunk_t *c; - svm_fifo_t *f = 0; + svm_fifo_shared_t *sf = 0; + fss = fsh_slice_get (fsh, slice_index); min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096); fl_index = fs_freelist_for_size (min_size); if (!fss_chunk_fl_index_is_valid (fss, fl_index)) return 0; - f = fsh_try_alloc_fifo_hdr (fsh, fss); - if (!f) + sf = fsh_try_alloc_fifo_hdr (fsh, fss); + if (!sf) return 0; c = fsh_try_alloc_chunk (fsh, fss, min_size); if (!c) { - f->next = fss->free_fifos; - fss->free_fifos = f; + fss_fifo_free_list_push (fsh, fss, sf); return 0; } - f->start_chunk = c; + sf->start_chunk = fs_chunk_sptr (fsh, c); while (c->next) - c = c->next; - f->end_chunk = c; - f->size = data_bytes; - f->fs_hdr = fsh; + c = fs_chunk_ptr (fsh, c->next); + sf->end_chunk = fs_chunk_sptr (fsh, c); + sf->size = data_bytes; + sf->slice_index = slice_index; - return f; + return sf; } svm_fifo_chunk_t * @@ -700,9 +759,9 @@ fsh_slice_collect_chunks (fifo_segment_header_t * fsh, while (c) { CLIB_MEM_UNPOISON (c, sizeof (*c)); - next = c->next; + next = fs_chunk_ptr (fsh, c->next); fl_index = fs_freelist_for_size (c->length); - fss_chunk_free_list_push (fss, fl_index, c); + fss_chunk_free_list_push (fsh, fss, fl_index, c); n_collect += fs_freelist_index_to_size (fl_index); c = next; } @@ -720,6 +779,46 @@ fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index, fsh_slice_collect_chunks (fsh, fss, c); } +svm_fifo_t * +fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index) +{ + fifo_slice_private_t *pfss = &fs->slices[slice_index]; + svm_fifo_t *f; + + f = clib_mem_bulk_alloc (pfss->fifos); + clib_memset (f, 0, sizeof (*f)); + return f; +} + +void +fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f, u32 slice_index) +{ + fifo_slice_private_t *pfss; + + if (CLIB_DEBUG) + clib_memset (f, 0xfc, sizeof (*f)); + + pfss = &fs->slices[slice_index]; + clib_mem_bulk_free (pfss->fifos, f); +} + +void +fifo_segment_cleanup (fifo_segment_t *fs) +{ + int slice_index; + svm_msg_q_t *mq = 0; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + clib_mem_bulk_destroy (fs->slices[slice_index].fifos); + + vec_free (fs->slices); + + vec_foreach (fs->mqs, mq) + vec_free (mq->rings); + + vec_free (fs->mqs); +} + /** * Allocate fifo in fifo segment */ @@ -728,7 +827,9 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype) { fifo_segment_header_t *fsh = fs->h; + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; svm_fifo_t *f = 0; ASSERT (slice_index < fs->n_slices); @@ -736,22 +837,26 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size)) return 0; - fss = fsh_slice_get (fsh, slice_index); - f = fs_try_alloc_fifo (fsh, fss, data_bytes); - if (!f) + sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes); + if (!sf) goto done; - f->slice_index = slice_index; + f = fs_fifo_alloc (fs, slice_index); + f->fs_hdr = fsh; + f->shr = sf; svm_fifo_init (f, data_bytes); + fss = fsh_slice_get (fsh, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track * only one. */ if (ftype == FIFO_SEGMENT_RX_FIFO) { - fss_fifo_add_active_list (fss, f); + pfss_fifo_add_active_list (pfss, f); f->flags |= SVM_FIFO_F_LL_TRACKED; } @@ -762,6 +867,22 @@ done: return (f); } +svm_fifo_t * +fifo_segment_alloc_fifo_w_offset (fifo_segment_t *fs, uword offset) +{ + svm_fifo_t *f = fs_fifo_alloc (fs, 0); + svm_fifo_shared_t *sf; + + sf = (svm_fifo_shared_t *) ((u8 *) fs->h + offset); + f->fs_hdr = fs->h; + f->shr = sf; + + f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->segment_index = SVM_FIFO_INVALID_INDEX; + f->refcnt = 1; + return f; +} + /** * Free fifo allocated in fifo segment */ @@ -769,94 +890,234 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh = fs->h; + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; ASSERT (f->refcnt > 0); if (--f->refcnt > 0) return; - fss = fsh_slice_get (fsh, f->slice_index); + /* + * Cleanup shared state + */ + + sf = f->shr; + fss = fsh_slice_get (fsh, sf->slice_index); + pfss = fs_slice_private_get (fs, sf->slice_index); + + /* Free fifo chunks */ + fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk)); + + sf->start_chunk = sf->end_chunk = 0; + sf->head_chunk = sf->tail_chunk = 0; + + /* Add to free list */ + fss_fifo_free_list_push (fsh, fss, sf); + + fss->virtual_mem -= svm_fifo_size (f); + + /* + * Cleanup private state + */ /* Remove from active list. Only rx fifos are tracked */ if (f->flags & SVM_FIFO_F_LL_TRACKED) { - fss_fifo_del_active_list (fss, f); + pfss_fifo_del_active_list (pfss, f); f->flags &= ~SVM_FIFO_F_LL_TRACKED; } - /* Free fifo chunks */ - fsh_slice_collect_chunks (fsh, fss, f->start_chunk); - - f->start_chunk = f->end_chunk = 0; - f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; - - /* not allocated on segment heap */ svm_fifo_free_chunk_lookup (f); svm_fifo_free_ooo_data (f); if (CLIB_DEBUG) { - f->master_session_index = ~0; + sf->master_session_index = ~0; f->master_thread_index = ~0; } - fss->virtual_mem -= svm_fifo_size (f); - - /* Add to free list */ - f->next = fss->free_fifos; + f->ooo_enq = f->ooo_deq = 0; f->prev = 0; - fss->free_fifos = f; + + fs_fifo_free (fs, f, f->shr->slice_index); fsh_active_fifos_update (fsh, -1); } void -fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f) +fifo_segment_free_client_fifo (fifo_segment_t *fs, svm_fifo_t *f) { + fs_fifo_free (fs, f, 0 /* clients attach fifos in slice 0 */); +} + +void +fifo_segment_detach_fifo (fifo_segment_t *fs, svm_fifo_t **f) +{ + fifo_slice_private_t *pfss; fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - u32 fl_index; + svm_fifo_t *of = *f; + u32 slice_index; - ASSERT (f->refcnt == 1); + slice_index = of->master_thread_index; + fss = fsh_slice_get (fs->h, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + fss->virtual_mem -= svm_fifo_size (of); + if (of->flags & SVM_FIFO_F_LL_TRACKED) + pfss_fifo_del_active_list (pfss, of); - fss = fsh_slice_get (fs->h, f->slice_index); - fss->virtual_mem -= svm_fifo_size (f); - if (f->flags & SVM_FIFO_F_LL_TRACKED) - fss_fifo_del_active_list (fss, f); + /* Collect chunks that were provided in return for those detached */ + fsh_slice_collect_chunks (fs->h, fss, of->chunks_at_attach); + of->chunks_at_attach = 0; - c = f->start_chunk; - while (c) + /* Collect hdr that was provided in return for the detached */ + fss_fifo_free_list_push (fs->h, fss, of->hdr_at_attach); + of->hdr_at_attach = 0; + + clib_mem_bulk_free (pfss->fifos, *f); + *f = 0; +} + +void +fifo_segment_attach_fifo (fifo_segment_t *fs, svm_fifo_t **f, u32 slice_index) +{ + svm_fifo_chunk_t *c, *nc, *pc = 0; + fifo_slice_private_t *pfss; + fifo_segment_slice_t *fss; + svm_fifo_t *nf, *of; + + nf = fs_fifo_alloc (fs, slice_index); + clib_memcpy_fast (nf, *f, sizeof (*nf)); + + fss = fsh_slice_get (fs->h, slice_index); + pfss = fs_slice_private_get (fs, slice_index); + fss->virtual_mem += svm_fifo_size (nf); + nf->next = nf->prev = 0; + if (nf->flags & SVM_FIFO_F_LL_TRACKED) + pfss_fifo_add_active_list (pfss, nf); + + /* Allocate shared hdr and chunks to be collected at detach in return + * for those that are being attached now */ + of = *f; + of->hdr_at_attach = fsh_try_alloc_fifo_hdr (fs->h, fss); + + c = fs_chunk_ptr (fs->h, nf->shr->start_chunk); + of->chunks_at_attach = pc = fsh_try_alloc_chunk (fs->h, fss, c->length); + + while ((c = fs_chunk_ptr (fs->h, c->next))) { - fl_index = fs_freelist_for_size (c->length); - clib_atomic_fetch_sub_rel (&fss->num_chunks[fl_index], 1); - c = c->next; + nc = fsh_try_alloc_chunk (fs->h, fss, c->length); + pc->next = fs_chunk_sptr (fs->h, nc); + pc = nc; + } + + nf->shr->slice_index = slice_index; + *f = nf; +} + +uword +fifo_segment_fifo_offset (svm_fifo_t *f) +{ + return (u8 *) f->shr - (u8 *) f->fs_hdr; +} + +svm_msg_q_t * +fifo_segment_msg_q_alloc (fifo_segment_t *fs, u32 mq_index, + svm_msg_q_cfg_t *cfg) +{ + fifo_segment_header_t *fsh = fs->h; + svm_msg_q_shared_t *smq; + svm_msg_q_t *mq; + void *base; + u32 size; + + if (!fs->mqs) + { + u32 n_mqs = clib_max (fs->h->n_mqs, 1); + vec_validate (fs->mqs, n_mqs - 1); + } + + size = svm_msg_q_size_to_alloc (cfg); + base = fsh_alloc_aligned (fsh, size, 8); + fsh->n_reserved_bytes += size; + + smq = svm_msg_q_init (base, cfg); + mq = vec_elt_at_index (fs->mqs, mq_index); + svm_msg_q_attach (mq, smq); + + return mq; +} + +svm_msg_q_t * +fifo_segment_msg_q_attach (fifo_segment_t *fs, uword offset, u32 mq_index) +{ + svm_msg_q_t *mq; + + if (!fs->mqs) + { + u32 n_mqs = clib_max (fs->h->n_mqs, 1); + vec_validate (fs->mqs, n_mqs - 1); + } + + mq = vec_elt_at_index (fs->mqs, mq_index); + + if (!mq->q.shr) + { + svm_msg_q_shared_t *smq; + smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset); + svm_msg_q_attach (mq, smq); } + + ASSERT (fifo_segment_msg_q_offset (fs, mq_index) == offset); + + return mq; } void -fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f, - u32 slice_index) +fifo_segment_msg_qs_discover (fifo_segment_t *fs, int *fds, u32 n_fds) { - fifo_segment_slice_t *fss; - svm_fifo_chunk_t *c; - u32 fl_index; + svm_msg_q_shared_t *smq; + u32 n_mqs, size, i; + uword offset = 0, n_alloced; + svm_msg_q_t *mq; - f->slice_index = slice_index; - fss = fsh_slice_get (fs->h, f->slice_index); - fss->virtual_mem += svm_fifo_size (f); - if (f->flags & SVM_FIFO_F_LL_TRACKED) - fss_fifo_add_active_list (fss, f); + n_mqs = fs->h->n_mqs; + if (n_fds && n_mqs != n_fds) + { + clib_warning ("expected %u fds got %u", n_mqs, n_fds); + return; + } - c = f->start_chunk; - while (c) + vec_validate (fs->mqs, n_mqs - 1); + n_alloced = fs->h->n_reserved_bytes - fs->h->start_byte_index; + ASSERT (n_alloced % n_mqs == 0); + size = n_alloced / n_mqs; + + offset = fs->h->start_byte_index; + for (i = 0; i < n_mqs; i++) { - fl_index = fs_freelist_for_size (c->length); - clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1); - c = c->next; + mq = vec_elt_at_index (fs->mqs, i); + smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset); + svm_msg_q_attach (mq, smq); + if (n_fds) + svm_msg_q_set_eventfd (mq, fds[i]); + offset += size; } } +uword +fifo_segment_msg_q_offset (fifo_segment_t *fs, u32 mq_index) +{ + svm_msg_q_t *mq = vec_elt_at_index (fs->mqs, mq_index); + + if (mq->q.shr == 0) + return ~0ULL; + + return (uword) ((u8 *) mq->q.shr - (u8 *) fs->h) - + sizeof (svm_msg_q_shared_t); +} + int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, u32 batch_size) @@ -964,18 +1225,18 @@ fifo_segment_num_fifos (fifo_segment_t * fs) } static u32 -fs_slice_num_free_fifos (fifo_segment_slice_t * fss) +fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - svm_fifo_t *f; + svm_fifo_shared_t *f; u32 count = 0; - f = fss->free_fifos; + f = fs_ptr (fsh, fss->free_fifos); if (f == 0) return 0; while (f) { - f = f->next; + f = fs_ptr (fsh, f->next); count++; } return count; @@ -992,13 +1253,14 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - count += fs_slice_num_free_fifos (fss); + count += fs_slice_num_free_fifos (fsh, fss); } return count; } static u32 -fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) +fs_slice_num_free_chunks (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, u32 size) { u32 count = 0, rounded_size, fl_index; svm_fifo_chunk_t *c; @@ -1009,13 +1271,13 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) { for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { - c = fss->free_chunks[i]; + c = fs_chunk_ptr (fsh, fss->free_chunks[i]); if (c == 0) continue; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } } @@ -1028,13 +1290,13 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) if (fl_index >= FS_CHUNK_VEC_LEN) return 0; - c = fss->free_chunks[fl_index]; + c = fs_chunk_ptr (fsh, fss->free_chunks[fl_index]); if (c == 0) return 0; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } return count; @@ -1051,7 +1313,7 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - count += fs_slice_num_free_chunks (fss, size); + count += fs_slice_num_free_chunks (fsh, fss, size); } return count; } @@ -1128,11 +1390,10 @@ fifo_segment_has_fifos (fifo_segment_t * fs) svm_fifo_t * fifo_segment_get_slice_fifo_list (fifo_segment_t * fs, u32 slice_index) { - fifo_segment_header_t *fsh = fs->h; - fifo_segment_slice_t *fss; + fifo_slice_private_t *pfss; - fss = fsh_slice_get (fsh, slice_index); - return fss->fifos; + pfss = fs_slice_private_get (fs, slice_index); + return pfss->active_fifos; } u8 @@ -1258,13 +1519,13 @@ format_fifo_segment (u8 * s, va_list * args) fss = fsh_slice_get (fsh, slice_index); for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { - c = fss->free_chunks[i]; + c = fs_chunk_ptr (fsh, fss->free_chunks[i]); if (c == 0 && fss->num_chunks[i] == 0) continue; count = 0; while (c) { - c = c->next; + c = fs_chunk_ptr (fsh, c->next); count++; } @@ -1298,8 +1559,8 @@ format_fifo_segment (u8 * s, va_list * args) format_memory_size, chunk_bytes, chunk_bytes, format_memory_size, est_chunk_bytes, est_chunk_bytes, format_memory_size, tracked_cached_bytes, tracked_cached_bytes); - s = format (s, "%Ufifo active: %u hdr free bytes: %U (%u) \n", - format_white_space, indent + 2, fsh->n_active_fifos, + s = format (s, "%Ufifo active: %u hdr free: %u bytes: %U (%u) \n", + format_white_space, indent + 2, fsh->n_active_fifos, free_fifos, format_memory_size, fifo_hdr, fifo_hdr); s = format (s, "%Usegment usage: %.2f%% (%U / %U) virt: %U status: %s\n", format_white_space, indent + 2, usage, format_memory_size,