#include <svm/svm_fifo_segment.h>
-svm_fifo_segment_main_t svm_fifo_segment_main;
-
static void
allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh,
u32 data_size_in_bytes, int chunk_size)
}
}
-static void
-preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
- svm_fifo_segment_create_args_t * a, u32 protect_size)
+/**
+ * Pre-allocates fifo pairs in fifo segment
+ *
+ * The number of fifos pre-allocated is the minimum of the requested number
+ * of pairs and the maximum number that fit within the segment. If the maximum
+ * is hit, the number of fifo pairs requested is updated by subtracting the
+ * number of fifos that have been successfully allocated.
+ */
+void
+svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
+ u32 rx_fifo_size, u32 tx_fifo_size,
+ u32 * n_fifo_pairs)
{
- svm_fifo_segment_header_t *fsh = s->h;
- u32 rx_fifo_size, tx_fifo_size, pairs_to_allocate;
u32 rx_rounded_data_size, tx_rounded_data_size, pair_size;
- svm_fifo_t *f;
- u8 *rx_fifo_space, *tx_fifo_space;
+ u32 rx_fifos_size, tx_fifos_size, pairs_to_allocate;
int rx_freelist_index, tx_freelist_index;
+ ssvm_shared_header_t *sh = s->ssvm.sh;
+ svm_fifo_segment_header_t *fsh = s->h;
+ u8 *rx_fifo_space, *tx_fifo_space;
+ uword space_available;
+ void *oldheap;
+ svm_fifo_t *f;
int i;
/* Parameter check */
- if (a->rx_fifo_size == 0 || a->tx_fifo_size == 0
- || a->preallocated_fifo_pairs == 0)
+ if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
return;
- if (a->rx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
- a->rx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
+ if (rx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
+ rx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
{
- clib_warning ("rx fifo_size out of range %d", a->rx_fifo_size);
+ clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
return;
}
- if (a->tx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
- a->tx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
+ if (tx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
+ tx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
{
- clib_warning ("tx fifo_size out of range %d", a->rx_fifo_size);
+ clib_warning ("tx fifo_size out of range %d", rx_fifo_size);
return;
}
- rx_rounded_data_size = (1 << (max_log2 (a->rx_fifo_size)));
-
- rx_freelist_index = max_log2 (a->rx_fifo_size)
+ rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size)));
+ rx_freelist_index = max_log2 (rx_fifo_size)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
-
- tx_rounded_data_size = (1 << (max_log2 (a->rx_fifo_size)));
-
- tx_freelist_index = max_log2 (a->tx_fifo_size)
+ tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size)));
+ tx_freelist_index = max_log2 (tx_fifo_size)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
/* Calculate space requirements */
pair_size = 2 * sizeof (*f) + rx_rounded_data_size + tx_rounded_data_size;
- if (protect_size)
- protect_size += mheap_bytes (s->ssvm.sh->heap);
- pairs_to_allocate =
- clib_min ((s->ssvm.ssvm_size - protect_size) / pair_size,
- a->preallocated_fifo_pairs);
- rx_fifo_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate;
- tx_fifo_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate;
+#if USE_DLMALLOC == 0
+ space_available = s->ssvm.ssvm_size - mheap_bytes (sh->heap);
+#else
+ space_available = s->ssvm.ssvm_size - mspace_usable_size (sh->heap);
+#endif
+
+ pairs_to_allocate = clib_min (space_available / pair_size, *n_fifo_pairs);
+ rx_fifos_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate;
+ tx_fifos_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate;
vec_validate_init_empty (fsh->free_fifos,
clib_max (rx_freelist_index, tx_freelist_index),
0);
+ oldheap = ssvm_push_heap (sh);
/* Allocate rx fifo space. May fail. */
rx_fifo_space = clib_mem_alloc_aligned_at_offset
- (rx_fifo_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
+ (rx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
0 /* os_out_of_memory */ );
/* Same for TX */
tx_fifo_space = clib_mem_alloc_aligned_at_offset
- (tx_fifo_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
+ (tx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
0 /* os_out_of_memory */ );
/* Make sure it worked. Clean up if it didn't... */
clib_mem_free (rx_fifo_space);
else
clib_warning ("rx fifo preallocation failure: size %d npairs %d",
- a->rx_fifo_size, a->preallocated_fifo_pairs);
+ rx_fifo_size, *n_fifo_pairs);
if (tx_fifo_space)
clib_mem_free (tx_fifo_space);
else
clib_warning ("tx fifo preallocation failure: size %d nfifos %d",
- a->tx_fifo_size, a->preallocated_fifo_pairs);
+ tx_fifo_size, *n_fifo_pairs);
+ ssvm_pop_heap (oldheap);
return;
}
}
/* Account for the pairs allocated */
- a->preallocated_fifo_pairs -= pairs_to_allocate;
+ *n_fifo_pairs -= pairs_to_allocate;
+ ssvm_pop_heap (oldheap);
}
/**
- * Create an svm fifo segment and initialize as master
+ * Initialize svm fifo segment shared header
*/
int
-svm_fifo_segment_create (svm_fifo_segment_create_args_t * a)
+svm_fifo_segment_init (svm_fifo_segment_private_t * s)
{
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
- svm_fifo_segment_private_t *s;
- ssvm_shared_header_t *sh;
svm_fifo_segment_header_t *fsh;
+ ssvm_shared_header_t *sh;
void *oldheap;
+
+ sh = s->ssvm.sh;
+ oldheap = ssvm_push_heap (sh);
+
+ fsh = clib_mem_alloc (sizeof (*fsh));
+ clib_memset (fsh, 0, sizeof (*fsh));
+ s->h = sh->opaque[0] = fsh;
+
+ ssvm_pop_heap (oldheap);
+
+ sh->ready = 1;
+ return (0);
+}
+
+/**
+ * Create an svm fifo segment and initialize as master
+ */
+int
+svm_fifo_segment_create (svm_fifo_segment_main_t * sm,
+ svm_fifo_segment_create_args_t * a)
+{
+ svm_fifo_segment_private_t *s;
int rv;
/* Allocate a fresh segment */
pool_get (sm->segments, s);
- memset (s, 0, sizeof (*s));
+ clib_memset (s, 0, sizeof (*s));
s->ssvm.ssvm_size = a->segment_size;
s->ssvm.i_am_master = 1;
/* Note: requested_va updated due to seg base addr randomization */
sm->next_baseva = s->ssvm.sh->ssvm_va + a->segment_size;
- sh = s->ssvm.sh;
- oldheap = ssvm_push_heap (sh);
-
- /* Set up svm_fifo_segment shared header */
- fsh = clib_mem_alloc (sizeof (*fsh));
- memset (fsh, 0, sizeof (*fsh));
- s->h = sh->opaque[0] = fsh;
- preallocate_fifo_pairs (s, a, a->seg_protected_space);
-
- ssvm_pop_heap (oldheap);
-
- sh->ready = 1;
+ svm_fifo_segment_init (s);
vec_add1 (a->new_segment_indices, s - sm->segments);
return (0);
}
* Create an svm fifo segment in process-private memory
*/
int
-svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t * a)
+svm_fifo_segment_create_process_private (svm_fifo_segment_main_t * sm,
+ svm_fifo_segment_create_args_t * a)
{
svm_fifo_segment_private_t *s;
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
ssvm_shared_header_t *sh;
- svm_fifo_segment_header_t *fsh;
- void *oldheap;
- u8 **heaps = 0;
- mheap_t *heap_header;
- int segment_count = 1;
u32 rnd_size = 0;
- int i;
+ u8 *heap;
+ u32 pagesize = clib_mem_get_page_size ();
- if (a->private_segment_count)
- {
- u8 *heap;
- u32 pagesize = clib_mem_get_page_size ();
- rnd_size = (a->segment_size + (pagesize - 1)) & ~pagesize;
+ pool_get (sm->segments, s);
+ clib_memset (s, 0, sizeof (*s));
- for (i = 0; i < a->private_segment_count; i++)
- {
- heap = mheap_alloc (0, rnd_size);
- if (heap == 0)
- {
- clib_unix_warning ("mheap alloc");
- return -1;
- }
- heap_header = mheap_header (heap);
- heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
- vec_add1 (heaps, heap);
- }
- segment_count = a->private_segment_count;
- }
+ rnd_size = (a->segment_size + (pagesize - 1)) & ~pagesize;
- /* Allocate segments */
- for (i = 0; i < segment_count; i++)
+#if USE_DLMALLOC == 0
+ heap = mheap_alloc (0, rnd_size);
+ if (heap == 0)
{
- pool_get (sm->segments, s);
- memset (s, 0, sizeof (*s));
-
- s->ssvm.ssvm_size = rnd_size;
- s->ssvm.i_am_master = 1;
- s->ssvm.my_pid = getpid ();
- s->ssvm.name = format (0, "%s%c", a->segment_name, 0);
- s->ssvm.requested_va = ~0;
-
- /* Allocate a [sic] shared memory header, in process memory... */
- sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES);
- s->ssvm.sh = sh;
-
- memset (sh, 0, sizeof (*sh));
- sh->heap = a->private_segment_count ? heaps[i] : clib_mem_get_heap ();
-
- /* Set up svm_fifo_segment shared header */
- fsh = clib_mem_alloc (sizeof (*fsh));
- memset (fsh, 0, sizeof (*fsh));
- fsh->flags = FIFO_SEGMENT_F_IS_PRIVATE;
- s->h = sh->opaque[0] = fsh;
- if (!a->private_segment_count)
- fsh->flags |= FIFO_SEGMENT_F_IS_MAIN_HEAP;
-
- if (a->private_segment_count)
- {
- if (i != 0)
- fsh->flags |= FIFO_SEGMENT_F_IS_PREALLOCATED;
- oldheap = clib_mem_get_heap ();
- clib_mem_set_heap (sh->heap);
- preallocate_fifo_pairs (s, a, i == 0 ? a->seg_protected_space : 0);
- clib_mem_set_heap (oldheap);
- }
- sh->ready = 1;
- vec_add1 (a->new_segment_indices, s - sm->segments);
+ clib_unix_warning ("mheap alloc");
+ pool_put (sm->segments, s);
+ return -1;
}
- vec_free (heaps);
+ {
+ mheap_t *heap_header;
+ heap_header = mheap_header (heap);
+ heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
+ }
+#else
+ heap = create_mspace (rnd_size, 1 /* locked */ );
+#endif
+
+ s->ssvm.ssvm_size = rnd_size;
+ s->ssvm.i_am_master = 1;
+ s->ssvm.my_pid = getpid ();
+ s->ssvm.name = format (0, "%s%c", a->segment_name, 0);
+ s->ssvm.requested_va = ~0;
+
+ /* Allocate a [sic] shared memory header, in process memory... */
+ sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES);
+ s->ssvm.sh = sh;
+
+ clib_memset (sh, 0, sizeof (*sh));
+ sh->heap = heap;
+
+ svm_fifo_segment_init (s);
+ vec_add1 (a->new_segment_indices, s - sm->segments);
+
return (0);
}
* Attach as slave to an svm fifo segment
*/
int
-svm_fifo_segment_attach (svm_fifo_segment_create_args_t * a)
+svm_fifo_segment_attach (svm_fifo_segment_main_t * sm,
+ svm_fifo_segment_create_args_t * a)
{
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
svm_fifo_segment_private_t *s;
int rv;
/* Allocate a fresh segment */
pool_get (sm->segments, s);
- memset (s, 0, sizeof (*s));
+ clib_memset (s, 0, sizeof (*s));
s->ssvm.ssvm_size = a->segment_size;
s->ssvm.my_pid = getpid ();
}
void
-svm_fifo_segment_delete (svm_fifo_segment_private_t * s)
+svm_fifo_segment_delete (svm_fifo_segment_main_t * sm,
+ svm_fifo_segment_private_t * s)
{
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
-
- if (s->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- {
- /* Don't try to free vpp's heap! */
- if (!(s->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
- mheap_free (s->ssvm.sh->heap);
- clib_mem_free (s->ssvm.sh);
- clib_mem_free (s->h);
- }
- else
- {
- ssvm_delete (&s->ssvm);
- }
- memset (s, 0xfe, sizeof (*s));
+ ssvm_delete (&s->ssvm);
+ clib_memset (s, 0xfe, sizeof (*s));
pool_put (sm->segments, s);
}
* Allocate fifo in svm segment
*/
svm_fifo_t *
-svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * s,
+svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * fs,
u32 data_size_in_bytes,
svm_fifo_segment_freelist_t list_index)
{
ssvm_shared_header_t *sh;
svm_fifo_segment_header_t *fsh;
- svm_fifo_t *f;
+ svm_fifo_t *f = 0;
void *oldheap;
int freelist_index;
freelist_index = max_log2 (data_size_in_bytes)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
- sh = s->ssvm.sh;
- fsh = (svm_fifo_segment_header_t *) sh->opaque[0];
-
+ sh = fs->ssvm.sh;
ssvm_lock_non_recursive (sh, 1);
- oldheap = ssvm_push_heap (sh);
+ fsh = (svm_fifo_segment_header_t *) sh->opaque[0];
switch (list_index)
{
case FIFO_SEGMENT_RX_FREELIST:
case FIFO_SEGMENT_TX_FREELIST:
vec_validate_init_empty (fsh->free_fifos, freelist_index, 0);
-
f = fsh->free_fifos[freelist_index];
- if (PREDICT_FALSE (f == 0))
+ if (PREDICT_FALSE (!f))
{
+ /* Preallocated and no fifo left. Don't even try */
+ if (fsh->flags & FIFO_SEGMENT_F_IS_PREALLOCATED)
+ goto done;
+
+ oldheap = ssvm_push_heap (sh);
allocate_new_fifo_chunk (fsh, data_size_in_bytes,
FIFO_SEGMENT_ALLOC_CHUNK_SIZE);
+ ssvm_pop_heap (oldheap);
f = fsh->free_fifos[freelist_index];
}
if (PREDICT_TRUE (f != 0))
{
fsh->free_fifos[freelist_index] = f->next;
/* (re)initialize the fifo, as in svm_fifo_create */
- memset (f, 0, sizeof (*f));
+ clib_memset (f, 0, sizeof (*f));
f->nitems = data_size_in_bytes;
f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
f->refcnt = 1;
f->freelist_index = freelist_index;
goto found;
}
- /* FALLTHROUGH */
+ break;
case FIFO_SEGMENT_FREELIST_NONE:
break;
break;
}
- /* Note: this can fail, in which case: create another segment */
+ /* Catch all that allocates just one fifo. Note: this can fail,
+ * in which case: create another segment */
+ oldheap = ssvm_push_heap (sh);
f = svm_fifo_create (data_size_in_bytes);
+ ssvm_pop_heap (oldheap);
if (PREDICT_FALSE (f == 0))
- {
- ssvm_pop_heap (oldheap);
- ssvm_unlock_non_recursive (sh);
- return (0);
- }
+ goto done;
f->freelist_index = freelist_index;
found:
}
fsh->n_active_fifos++;
- ssvm_pop_heap (oldheap);
+done:
ssvm_unlock_non_recursive (sh);
return (f);
}
}
void
-svm_fifo_segment_init (u64 baseva, u32 timeout_in_seconds)
+svm_fifo_segment_main_init (svm_fifo_segment_main_t * sm, u64 baseva,
+ u32 timeout_in_seconds)
{
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
-
sm->next_baseva = baseva;
sm->timeout_in_seconds = timeout_in_seconds;
}
u32
-svm_fifo_segment_index (svm_fifo_segment_private_t * s)
+svm_fifo_segment_index (svm_fifo_segment_main_t * sm,
+ svm_fifo_segment_private_t * s)
{
- return s - svm_fifo_segment_main.segments;
+ return s - sm->segments;
}
/**
* Retrieve svm segments pool. Used only for debug purposes.
*/
svm_fifo_segment_private_t *
-svm_fifo_segment_segments_pool (void)
+svm_fifo_segment_segments_pool (svm_fifo_segment_main_t * sm)
{
- svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
return sm->segments;
}
freelist_index = max_log2 (rounded_data_size)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
- if (freelist_index > vec_len (fsh->free_fifos))
+ if (freelist_index >= vec_len (fsh->free_fifos))
return 0;
f = fsh->free_fifos[freelist_index];
svm_fifo_segment_info (svm_fifo_segment_private_t * seg, uword * address,
u64 * size)
{
- if (seg->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
+ if (ssvm_type (&seg->ssvm) == SSVM_SEGMENT_PRIVATE)
{
+#if USE_DLMALLOC == 0
mheap_t *heap_header;
*address = pointer_to_uword (seg->ssvm.sh->heap);
heap_header = mheap_header (seg->ssvm.sh->heap);
*size = heap_header->max_size;
+#else
+ mspace_get_address_and_size (seg->ssvm.sh->heap,
+ (unsigned long long *) address,
+ (unsigned long long *) size);
+#endif
}
else
{
}
}
-mheap_t *
+void *
svm_fifo_segment_heap (svm_fifo_segment_private_t * seg)
{
return seg->ssvm.sh->heap;
sp = va_arg (*args, svm_fifo_segment_private_t *);
ssvm_segment_type_t st = ssvm_type (&sp->ssvm);
- if ((sp->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- && !(sp->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
+ if (st == SSVM_SEGMENT_PRIVATE)
s = format (s, "%s", "private-heap");
- else if ((sp->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- && (sp->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
- s = format (s, "%s", "main-heap");
else if (st == SSVM_SEGMENT_MEMFD)
s = format (s, "%s", "memfd");
else if (st == SSVM_SEGMENT_SHM)
{
svm_fifo_segment_private_t *sp
= va_arg (*args, svm_fifo_segment_private_t *);
- int verbose = va_arg (*args, int);
+ int verbose __attribute__ ((unused)) = va_arg (*args, int);
svm_fifo_segment_header_t *fsh = sp->h;
u32 count, indent;
svm_fifo_t *f;
int i;
indent = format_get_indent (s) + 2;
+#if USE_DLMALLOC == 0
s = format (s, "%U segment heap: %U\n", format_white_space, indent,
format_mheap, svm_fifo_segment_heap (sp), verbose);
s = format (s, "%U segment has %u active fifos\n",
format_white_space, indent, svm_fifo_segment_num_fifos (sp));
+#endif
for (i = 0; i < vec_len (fsh->free_fifos); i++)
{