+ fss_chunk_free_list_push_list (fsh, fss, fl_index, head, tail);
+ fss->num_chunks[fl_index] += batch_size;
+ fss_fl_chunk_bytes_add (fss, total_chunk_bytes);
+ fsh_cached_bytes_add (fsh, total_chunk_bytes);
+
+ return 0;
+}
+
+static int
+fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
+ fifo_segment_slice_t * fss,
+ u32 fl_index, u32 batch_size)
+{
+ if (fsh_try_alloc_fifo_hdr_batch (fsh, fss, batch_size))
+ return 0;
+ return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size);
+}
+
+static svm_fifo_shared_t *
+fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
+{
+ svm_fifo_shared_t *sf;
+
+ if (!fss->free_fifos)
+ {
+ if (fsh_try_alloc_fifo_hdr_batch (fsh, fss,
+ FIFO_SEGMENT_ALLOC_BATCH_SIZE))
+ return 0;
+ }
+
+ sf = fss_fifo_free_list_pop (fsh, fss);
+ clib_memset (sf, 0, sizeof (*sf));
+
+ return sf;
+}
+
+static svm_fifo_chunk_t *
+fsh_try_alloc_chunk (fifo_segment_header_t * fsh,
+ fifo_segment_slice_t * fss, u32 data_bytes)
+{
+ svm_fifo_chunk_t *c;
+ u32 fl_index;
+
+ fl_index = fs_freelist_for_size (data_bytes);
+
+free_list:
+ c = fss_chunk_free_list_pop (fsh, fss, fl_index);
+ if (c)
+ {
+ c->next = 0;
+ fss_fl_chunk_bytes_sub (fss, fs_freelist_index_to_size (fl_index));
+ fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index));
+ }
+ else
+ {
+ u32 chunk_size, batch = FIFO_SEGMENT_ALLOC_BATCH_SIZE;
+ uword n_free;
+
+ chunk_size = fs_freelist_index_to_size (fl_index);
+ n_free = fsh_n_free_bytes (fsh);
+
+ if (chunk_size <= n_free)
+ {
+ batch = chunk_size * batch <= n_free ? batch : 1;
+ if (!fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch))
+ goto free_list;
+ }
+ /* Failed to allocate larger chunk, try to allocate multi-chunk
+ * that is close to what was actually requested */
+ if (data_bytes <= fss_fl_chunk_bytes (fss))
+ {
+ c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
+ if (c)
+ goto done;
+ batch = n_free / FIFO_SEGMENT_MIN_FIFO_SIZE;
+ if (!batch || fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
+ goto done;
+ }
+ if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
+ {
+ u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE;
+
+ batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
+ batch = clib_min (batch + 1, n_free / min_size);
+ if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
+ goto done;
+ c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
+ }
+ }
+
+done:
+
+ return c;
+}
+
+/**
+ * Try to allocate new fifo
+ *
+ * Tries the following steps in order:
+ * - grab fifo and chunk from freelists
+ * - batch fifo and chunk allocation
+ * - single fifo allocation
+ * - grab multiple fifo chunks from freelists
+ */
+static svm_fifo_shared_t *
+fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
+{
+ fifo_segment_slice_t *fss;
+ u32 fl_index, min_size;
+ svm_fifo_chunk_t *c;
+ svm_fifo_shared_t *sf = 0;
+
+ fss = fsh_slice_get (fsh, slice_index);
+ min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
+ fl_index = fs_freelist_for_size (min_size);
+
+ if (!fss_chunk_fl_index_is_valid (fss, fl_index))
+ return 0;
+
+ sf = fsh_try_alloc_fifo_hdr (fsh, fss);
+ if (!sf)
+ return 0;
+
+ c = fsh_try_alloc_chunk (fsh, fss, min_size);
+ if (!c)
+ {
+ fss_fifo_free_list_push (fsh, fss, sf);
+ return 0;
+ }
+
+ sf->start_chunk = fs_chunk_sptr (fsh, c);
+ while (c->next)
+ c = fs_chunk_ptr (fsh, c->next);
+ sf->end_chunk = fs_chunk_sptr (fsh, c);
+ sf->size = data_bytes;
+ sf->slice_index = slice_index;
+
+ return sf;
+}
+
+svm_fifo_chunk_t *
+fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size)
+{
+ fifo_segment_slice_t *fss;
+ svm_fifo_chunk_t *c;
+
+ fss = fsh_slice_get (fsh, slice_index);
+ c = fsh_try_alloc_chunk (fsh, fss, chunk_size);
+
+ return c;
+}
+
+static void
+fsh_slice_collect_chunks (fifo_segment_header_t * fsh,
+ fifo_segment_slice_t * fss, svm_fifo_chunk_t * c)
+{
+ u32 n_collect = 0, fl_index;
+ svm_fifo_chunk_t *next;
+
+ while (c)
+ {
+ CLIB_MEM_UNPOISON (c, sizeof (*c));
+ next = fs_chunk_ptr (fsh, c->next);
+ fl_index = fs_freelist_for_size (c->length);
+ fss_chunk_free_list_push (fsh, fss, fl_index, c);
+ n_collect += fs_freelist_index_to_size (fl_index);
+ c = next;
+ }
+
+ fss_fl_chunk_bytes_add (fss, n_collect);
+ fsh_cached_bytes_add (fsh, n_collect);
+}
+
+void
+fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index,
+ svm_fifo_chunk_t * c)
+{
+ fifo_segment_slice_t *fss;
+ fss = fsh_slice_get (fsh, slice_index);
+ fsh_slice_collect_chunks (fsh, fss, c);
+}
+
+svm_fifo_t *
+fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index)
+{
+ fifo_slice_private_t *pfss = &fs->slices[slice_index];
+ svm_fifo_t *f;
+
+ f = clib_mem_bulk_alloc (pfss->fifos);
+ clib_memset (f, 0, sizeof (*f));
+ return f;
+}
+
+void
+fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f, u32 slice_index)
+{
+ fifo_slice_private_t *pfss;
+
+ if (CLIB_DEBUG)
+ clib_memset (f, 0xfc, sizeof (*f));
+
+ pfss = &fs->slices[slice_index];
+ clib_mem_bulk_free (pfss->fifos, f);
+}
+
+void
+fifo_segment_cleanup (fifo_segment_t *fs)
+{
+ int slice_index;
+ svm_msg_q_t *mq = 0;
+
+ for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
+ clib_mem_bulk_destroy (fs->slices[slice_index].fifos);
+
+ vec_free (fs->slices);
+
+ vec_foreach (fs->mqs, mq)
+ svm_msg_q_cleanup (mq);
+
+ vec_free (fs->mqs);
+}
+
+/**
+ * Allocate fifo in fifo segment
+ */
+svm_fifo_t *
+fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
+ u32 data_bytes, fifo_segment_ftype_t ftype)
+{
+ fifo_segment_header_t *fsh = fs->h;
+ fifo_slice_private_t *pfss;
+ fifo_segment_slice_t *fss;
+ svm_fifo_shared_t *sf;
+ svm_fifo_t *f = 0;
+
+ ASSERT (slice_index < fs->n_slices);
+
+ if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size))
+ return 0;
+
+ sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes);
+ if (!sf)
+ goto done;
+
+ f = fs_fifo_alloc (fs, slice_index);
+ f->fs_hdr = fsh;
+ f->shr = sf;
+
+ svm_fifo_init (f, data_bytes);
+
+ fss = fsh_slice_get (fsh, slice_index);
+ pfss = fs_slice_private_get (fs, slice_index);
+
+ /* If rx fifo type add to active fifos list. When cleaning up segment,
+ * we need a list of active sessions that should be disconnected. Since
+ * both rx and tx fifos keep pointers to the session, it's enough to track
+ * only one. */
+ if (ftype == FIFO_SEGMENT_RX_FIFO)
+ {
+ pfss_fifo_add_active_list (pfss, f);
+ f->flags |= SVM_FIFO_F_LL_TRACKED;
+ }
+
+ fsh_active_fifos_update (fsh, 1);
+ fss->virtual_mem += svm_fifo_size (f);
+
+done:
+ return (f);
+}
+
+svm_fifo_t *
+fifo_segment_alloc_fifo_w_offset (fifo_segment_t *fs, uword offset)
+{
+ svm_fifo_t *f = fs_fifo_alloc (fs, 0);
+ svm_fifo_shared_t *sf;
+
+ sf = (svm_fifo_shared_t *) ((u8 *) fs->h + offset);
+ f->fs_hdr = fs->h;
+ f->shr = sf;
+
+ f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
+ f->refcnt = 1;
+ return f;
+}
+
+/**
+ * Free fifo allocated in fifo segment
+ */
+void
+fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
+{
+ fifo_segment_header_t *fsh = fs->h;
+ fifo_slice_private_t *pfss;
+ fifo_segment_slice_t *fss;
+ svm_fifo_shared_t *sf;
+
+ ASSERT (f->refcnt > 0);
+
+ if (--f->refcnt > 0)
+ return;
+
+ /*
+ * Cleanup shared state
+ */
+
+ sf = f->shr;
+ fss = fsh_slice_get (fsh, sf->slice_index);
+ pfss = fs_slice_private_get (fs, sf->slice_index);
+
+ /* Free fifo chunks */
+ fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk));
+
+ sf->start_chunk = sf->end_chunk = 0;
+ sf->head_chunk = sf->tail_chunk = 0;
+
+ /* Add to free list */
+ fss_fifo_free_list_push (fsh, fss, sf);
+
+ fss->virtual_mem -= svm_fifo_size (f);
+
+ /*
+ * Cleanup private state
+ */
+
+ /* Remove from active list. Only rx fifos are tracked */
+ if (f->flags & SVM_FIFO_F_LL_TRACKED)
+ {
+ pfss_fifo_del_active_list (pfss, f);
+ f->flags &= ~SVM_FIFO_F_LL_TRACKED;
+ }
+
+ svm_fifo_free_chunk_lookup (f);
+ svm_fifo_free_ooo_data (f);
+
+ if (CLIB_DEBUG)
+ {
+ sf->master_session_index = ~0;
+ f->master_thread_index = ~0;
+ }
+
+ f->ooo_enq = f->ooo_deq = 0;
+ f->prev = 0;
+
+ fs_fifo_free (fs, f, f->shr->slice_index);
+
+ fsh_active_fifos_update (fsh, -1);
+}
+
+void
+fifo_segment_free_client_fifo (fifo_segment_t *fs, svm_fifo_t *f)
+{
+ fs_fifo_free (fs, f, 0 /* clients attach fifos in slice 0 */);
+}
+
+void
+fifo_segment_detach_fifo (fifo_segment_t *fs, svm_fifo_t **f)
+{
+ fifo_slice_private_t *pfss;
+ fifo_segment_slice_t *fss;
+ svm_fifo_t *of = *f;
+ u32 slice_index;
+
+ slice_index = of->master_thread_index;
+ fss = fsh_slice_get (fs->h, slice_index);
+ pfss = fs_slice_private_get (fs, slice_index);
+ fss->virtual_mem -= svm_fifo_size (of);
+ if (of->flags & SVM_FIFO_F_LL_TRACKED)
+ pfss_fifo_del_active_list (pfss, of);
+
+ /* Collect chunks that were provided in return for those detached */
+ fsh_slice_collect_chunks (fs->h, fss, of->chunks_at_attach);
+ of->chunks_at_attach = 0;
+
+ /* Collect hdr that was provided in return for the detached */
+ fss_fifo_free_list_push (fs->h, fss, of->hdr_at_attach);
+ of->hdr_at_attach = 0;
+
+ clib_mem_bulk_free (pfss->fifos, *f);
+ *f = 0;
+}
+
+void
+fifo_segment_attach_fifo (fifo_segment_t *fs, svm_fifo_t **f, u32 slice_index)
+{
+ svm_fifo_chunk_t *c, *nc, *pc = 0;
+ fifo_slice_private_t *pfss;
+ fifo_segment_slice_t *fss;
+ svm_fifo_t *nf, *of;
+
+ nf = fs_fifo_alloc (fs, slice_index);
+ clib_memcpy_fast (nf, *f, sizeof (*nf));
+
+ fss = fsh_slice_get (fs->h, slice_index);
+ pfss = fs_slice_private_get (fs, slice_index);
+ fss->virtual_mem += svm_fifo_size (nf);
+ nf->next = nf->prev = 0;
+ if (nf->flags & SVM_FIFO_F_LL_TRACKED)
+ pfss_fifo_add_active_list (pfss, nf);
+
+ /* Allocate shared hdr and chunks to be collected at detach in return
+ * for those that are being attached now */
+ of = *f;
+ of->hdr_at_attach = fsh_try_alloc_fifo_hdr (fs->h, fss);
+
+ c = fs_chunk_ptr (fs->h, nf->shr->start_chunk);
+ of->chunks_at_attach = pc = fsh_try_alloc_chunk (fs->h, fss, c->length);
+
+ while ((c = fs_chunk_ptr (fs->h, c->next)))
+ {
+ nc = fsh_try_alloc_chunk (fs->h, fss, c->length);
+ pc->next = fs_chunk_sptr (fs->h, nc);
+ pc = nc;
+ }
+
+ nf->shr->slice_index = slice_index;
+ *f = nf;
+}
+
+uword
+fifo_segment_fifo_offset (svm_fifo_t *f)
+{
+ return (u8 *) f->shr - (u8 *) f->fs_hdr;
+}
+
+svm_fifo_chunk_t *
+fifo_segment_alloc_chunk_w_slice (fifo_segment_t *fs, u32 slice_index,
+ u32 chunk_size)
+{
+ fifo_segment_header_t *fsh = fs->h;
+ fifo_segment_slice_t *fss;
+
+ fss = fsh_slice_get (fsh, slice_index);
+ return fsh_try_alloc_chunk (fsh, fss, chunk_size);
+}
+
+void
+fifo_segment_collect_chunk (fifo_segment_t *fs, u32 slice_index,
+ svm_fifo_chunk_t *c)
+{
+ fsh_collect_chunks (fs->h, slice_index, c);
+}