fifo_segment_create_args_t _a, *a = &_a;
fifo_segment_main_t *sm = &segment_main;
u8 *test_data, *retrieved_data = 0;
- fifo_segment_t *sp;
+ fifo_segment_t *fs;
svm_fifo_t *f;
int rv;
SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
- sp = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
- f = fifo_segment_alloc_fifo (sp, 4096, FIFO_SEGMENT_RX_FIFO);
+ fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
+ f = fifo_segment_alloc_fifo (fs, 4096, FIFO_SEGMENT_RX_FIFO);
SFIFO_TEST (f != 0, "svm_fifo_segment_alloc_fifo");
vec_free (test_data);
vec_free (retrieved_data);
- fifo_segment_free_fifo (sp, f);
- fifo_segment_delete (sm, sp);
+ vec_free (a->new_segment_indices);
+ fifo_segment_free_fifo (fs, f);
+ fifo_segment_delete (sm, fs);
+ return 0;
+}
+
+static int
+sfifo_test_fifo_segment_fifo_grow (int verbose)
+{
+ fifo_segment_main_t *sm = &segment_main;
+ fifo_segment_create_args_t _a, *a = &_a;
+ int rv, fifo_size = 4096, n_chunks;
+ fifo_segment_t *fs;
+ svm_fifo_t *f;
+
+ clib_memset (a, 0, sizeof (*a));
+ a->segment_name = "fifo-test1";
+ a->segment_size = 256 << 10;
+
+ rv = fifo_segment_create (sm, a);
+
+ SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
+
+ /*
+ * Alloc and grow fifo
+ */
+ fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
+ f = fifo_segment_alloc_fifo (fs, fifo_size, FIFO_SEGMENT_RX_FIFO);
+
+ SFIFO_TEST (f != 0, "svm_fifo_segment_alloc_fifo");
+
+ fifo_segment_grow_fifo (fs, f, fifo_size);
+ SFIFO_TEST (f->size == 2 * fifo_size, "fifo size should be %u is %u",
+ 2 * fifo_size, f->size);
+
+ fifo_segment_grow_fifo (fs, f, 16 * fifo_size);
+ SFIFO_TEST (f->size == 18 * fifo_size, "fifo size should be %u is %u",
+ 18 * fifo_size, f->size);
+
+ /*
+ * Free and test free list size
+ */
+ fifo_segment_free_fifo (fs, f);
+
+ n_chunks = fifo_segment_num_free_chunks (fs, fifo_size);
+ SFIFO_TEST (n_chunks == 1, "free 2^10B chunks should be %u is %u", 1,
+ n_chunks);
+ n_chunks = fifo_segment_num_free_chunks (fs, 16 * fifo_size);
+ SFIFO_TEST (n_chunks == 1, "free 2^14B chunks should be %u is %u", 1,
+ n_chunks);
+ n_chunks = fifo_segment_num_free_chunks (fs, ~0);
+ SFIFO_TEST (n_chunks == 2, "free chunks should be %u is %u", 2, n_chunks);
+
+ /*
+ * Realloc fifo
+ */
+ f = fifo_segment_alloc_fifo (fs, fifo_size, FIFO_SEGMENT_RX_FIFO);
+
+ fifo_segment_grow_fifo (fs, f, fifo_size);
+ n_chunks = fifo_segment_num_free_chunks (fs, fifo_size);
+ SFIFO_TEST (n_chunks == 0, "free 2^10B chunks should be %u is %u", 0,
+ n_chunks);
+
+ fifo_segment_grow_fifo (fs, f, 16 * fifo_size);
+ SFIFO_TEST (n_chunks == 0, "free 2^14B chunks should be %u is %u", 0,
+ n_chunks);
+ n_chunks = fifo_segment_num_free_chunks (fs, ~0);
+ SFIFO_TEST (n_chunks == 0, "free chunks should be %u is %u", 0, n_chunks);
+
+ /*
+ * Free again
+ */
+ fifo_segment_free_fifo (fs, f);
+ n_chunks = fifo_segment_num_free_chunks (fs, ~0);
+ SFIFO_TEST (n_chunks == 2, "free chunks should be %u is %u", 2, n_chunks);
+
+ /*
+ * Cleanup
+ */
+ fifo_segment_delete (sm, fs);
+ vec_free (a->new_segment_indices);
return 0;
}
SFIFO_TEST (!rv, "svm_fifo_segment_attach returned %d", rv);
sp = fifo_segment_get_segment (sm, a->new_segment_indices[0]);
+ vec_free (a->new_segment_indices);
sh = sp->ssvm.sh;
fsh = (fifo_segment_header_t *) sh->opaque[0];
result = (u32 *) f->head_chunk->data;
SFIFO_TEST (*result == 0, "slave reported no error");
+ vec_free (a->new_segment_indices);
vec_free (test_data);
fifo_segment_free_fifo (sp, f);
fifo_segment_delete (sm, sp);
if ((rv = sfifo_test_fifo_segment_mempig (verbose)))
return -1;
}
+ else if (unformat (input, "grow fifo"))
+ {
+ if ((rv = sfifo_test_fifo_segment_fifo_grow (verbose)))
+ return -1;
+ }
else if (unformat (input, "all"))
{
if ((rv = sfifo_test_fifo_segment_hello_world (verbose)))
return -1;
if ((rv = sfifo_test_fifo_segment_mempig (verbose)))
return -1;
+ if ((rv = sfifo_test_fifo_segment_fifo_grow (verbose)))
+ return -1;
/* Pretty slow so avoid running it always
if ((rv = sfifo_test_fifo_segment_master_slave (verbose)))
return -1;
}
static inline int
-fs_fifo_size_is_valid (u32 size)
+fs_chunk_size_is_valid (u32 size)
{
/*
* 4K minimum. It's not likely that anything good will happen
void *oldheap;
int fl_index;
- if (!fs_fifo_size_is_valid (data_bytes))
+ if (!fs_chunk_size_is_valid (data_bytes))
{
clib_warning ("fifo size out of range %d", data_bytes);
return 0;
sh = fs->ssvm.sh;
ssvm_lock_non_recursive (sh, 1);
- fsh = (fifo_segment_header_t *) sh->opaque[0];
+ fsh = fs->h;
vec_validate_init_empty (fsh->free_fifos, fl_index, 0);
f = fsh->free_fifos[fl_index];
* Free fifo allocated in fifo segment
*/
void
-fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
+fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
{
fifo_segment_header_t *fsh;
ssvm_shared_header_t *sh;
if (--f->refcnt > 0)
return;
- sh = s->ssvm.sh;
- fsh = (fifo_segment_header_t *) sh->opaque[0];
+ sh = fs->ssvm.sh;
+ fsh = fs->h;
fl_index = f->freelist_index;
ASSERT (fl_index < vec_len (fsh->free_fifos));
f->prev = 0;
fsh->free_fifos[fl_index] = f;
+ /* If fifo has more chunks, free them */
+ if (f->flags & SVM_FIFO_F_MULTI_CHUNK)
+ {
+ svm_fifo_chunk_t *cur, *next;
+ void *oldheap;
+
+ next = f->start_chunk->next;
+ while (next != f->start_chunk)
+ {
+ cur = next;
+ next = next->next;
+ fl_index = fs_free_list_for_size (cur->length);
+ cur->next = fsh->free_chunks[fl_index];
+ fsh->free_chunks[fl_index] = cur;
+ }
+ oldheap = ssvm_push_heap (sh);
+ svm_fifo_free_chunk_lookup (f);
+ ssvm_pop_heap (oldheap);
+ }
+
+ /* not allocated on segment heap */
+ svm_fifo_free_ooo_data (f);
+
if (CLIB_DEBUG)
{
f->master_session_index = ~0;
if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
return;
- if (!fs_fifo_size_is_valid (rx_fifo_size))
+ if (!fs_chunk_size_is_valid (rx_fifo_size))
{
clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
return;
}
- if (!fs_fifo_size_is_valid (tx_fifo_size))
+ if (!fs_chunk_size_is_valid (tx_fifo_size))
{
clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
return;
ssvm_pop_heap (oldheap);
}
+int
+fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size)
+{
+ ssvm_shared_header_t *sh;
+ svm_fifo_chunk_t *c;
+ void *oldheap;
+ int fl_index;
+
+ if (!fs_chunk_size_is_valid (chunk_size))
+ {
+ clib_warning ("chunk size out of range %d", chunk_size);
+ return 0;
+ }
+
+ fl_index = fs_free_list_for_size (chunk_size);
+
+ sh = fs->ssvm.sh;
+ ssvm_lock_non_recursive (sh, 1);
+
+ vec_validate_init_empty (fs->h->free_chunks, fl_index, 0);
+ c = fs->h->free_chunks[fl_index];
+
+ oldheap = ssvm_push_heap (sh);
+
+ if (!c)
+ {
+ c = svm_fifo_chunk_alloc (chunk_size);
+ if (!c)
+ {
+ ssvm_pop_heap (oldheap);
+ return -1;
+ }
+ }
+ else
+ {
+ fs->h->free_chunks[fl_index] = c->next;
+ }
+
+ svm_fifo_add_chunk (f, c);
+
+ ssvm_pop_heap (oldheap);
+ ssvm_unlock_non_recursive (sh);
+ return 0;
+}
+
/**
* Get number of active fifos
*/
return count;
}
+u32
+fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
+{
+ u32 count = 0, rounded_size, fl_index;
+ fifo_segment_header_t *fsh;
+ svm_fifo_chunk_t *c;
+ int i;
+
+ fsh = fs->h;
+
+ /* Count all free chunks? */
+ if (size == ~0)
+ {
+ for (i = 0; i < vec_len (fsh->free_chunks); i++)
+ {
+ c = fsh->free_chunks[i];
+ if (c == 0)
+ continue;
+
+ while (c)
+ {
+ c = c->next;
+ count++;
+ }
+ }
+ return count;
+ }
+
+ rounded_size = (1 << (max_log2 (size)));
+ fl_index = fs_free_list_for_size (rounded_size);
+
+ if (fl_index >= vec_len (fsh->free_chunks))
+ return 0;
+
+ c = fsh->free_chunks[fl_index];
+ if (c == 0)
+ return 0;
+
+ while (c)
+ {
+ c = c->next;
+ count++;
+ }
+ return count;
+}
+
u8
fifo_segment_has_fifos (fifo_segment_t * fs)
{
typedef struct
{
- svm_fifo_t *fifos; /**< Linked list of active RX fifos */
- svm_fifo_t **free_fifos; /**< Freelists, by fifo size */
- u32 n_active_fifos; /**< Number of active fifos */
- u8 flags; /**< Segment flags */
+ svm_fifo_t *fifos; /**< Linked list of active RX fifos */
+ svm_fifo_t **free_fifos; /**< Freelists by fifo size */
+ svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */
+ u32 n_active_fifos; /**< Number of active fifos */
+ u8 flags; /**< Segment flags */
} fifo_segment_header_t;
typedef struct
/**
* Allocate fifo in fifo segment
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param data_bytes size of default fifo chunk in bytes
* @param ftype fifo type @ref fifo_segment_ftype_t
* @return new fifo or 0 if alloc failed
/**
* Free fifo allocated in fifo segment
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param f fifo to be freed
*/
void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
* is hit, the number of fifo pairs requested is updated by subtracting the
* number of fifos that have been successfully allocated.
*
- * @param fs fifo segment
+ * @param fs fifo segment for fifo
* @param rx_fifo_size data size of rx fifos
* @param tx_fifo_size data size of tx fifos
* @param n_fifo_pairs number of pairs requested. Prior to returning, this
u32 rx_fifo_size,
u32 tx_fifo_size,
u32 * n_fifo_pairs);
+/**
+ * Grow fifo size by adding an additional chunk of memory
+ *
+ * @param fs fifo segment for fifo
+ * @param f fifo to be grown
+ * @param chunk_size number of bytes to be added to fifo
+ * @return 0 on success or a negative number otherwise
+ */
+int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f,
+ u32 chunk_size);
u8 fifo_segment_has_fifos (fifo_segment_t * fs);
svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs);
u32 fifo_segment_num_fifos (fifo_segment_t * fs);
u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes);
+/**
+ * Find number of free chunks of given size
+ *
+ * @param fs fifo segment
+ * @param size chunk size of interest or ~0 if all should be counted
+ * @return number of chunks of given size
+ */
+u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size);
void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva,
u32 timeout_in_seconds);
f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
}
-/** create an svm fifo, in the current heap. Fails vs blow up the process */
+/**
+ * Creates a fifo in the current heap. Fails vs blow up the process
+ */
svm_fifo_t *
svm_fifo_create (u32 data_size_in_bytes)
{
return f;
}
+/**
+ * Creates a fifo chunk in the current heap
+ */
+svm_fifo_chunk_t *
+svm_fifo_chunk_alloc (u32 size)
+{
+ svm_fifo_chunk_t *c;
+ u32 rounded_size;
+
+ /* round chunk size to the next highest power-of-two */
+ rounded_size = (1 << (max_log2 (size)));
+ c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size,
+ CLIB_CACHE_LINE_BYTES);
+ if (c == 0)
+ return 0;
+
+ clib_memset (c, 0, sizeof (*c));
+ c->length = rounded_size;
+ return c;
+}
+
static inline void
svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c)
{
return 0;
}
+void
+svm_fifo_free_chunk_lookup (svm_fifo_t * f)
+{
+ rb_tree_free_nodes (&f->chunk_lookup);
+}
+
+void
+svm_fifo_free_ooo_data (svm_fifo_t * f)
+{
+ pool_free (f->ooo_segments);
+}
+
void
svm_fifo_free (svm_fifo_t * f)
{
if (--f->refcnt == 0)
{
- pool_free (f->ooo_segments);
+ /* ooo data is not allocated on segment heap */
+ svm_fifo_free_chunk_lookup (f);
clib_mem_free (f);
}
}
svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
void svm_fifo_init (svm_fifo_t * f, u32 size);
+
+/**
+ * Allocate a fifo chunk on heap
+ *
+ * If the chunk is allocated on a fifo segment, this should be called
+ * with the segment's heap pushed.
+ *
+ * @param size chunk size in bytes. Will be rounded to the next highest
+ * power-of-two
+ * @return new chunk or 0 if alloc failed
+ */
+svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size);
+
/**
* Grow fifo size by adding chunk to chunk list
*
*/
void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c);
void svm_fifo_free (svm_fifo_t * f);
+/**
+ * Cleanup fifo chunk lookup rb tree
+ *
+ * The rb tree is allocated in segment heap so this should be called
+ * with it pushed.
+ *
+ * @param f fifo to cleanup
+ */
+void svm_fifo_free_chunk_lookup (svm_fifo_t * f);
+/**
+ * Cleanup fifo ooo data
+ *
+ * The ooo data is allocated in producer process memory. The fifo
+ * segment heap should not be pushed.
+ *
+ * @param f fifo to cleanup
+ */
+void svm_fifo_free_ooo_data (svm_fifo_t * f);
int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
const u8 * copy_from_here);
void
segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo)
{
- fifo_segment_t *fs;
segment_manager_t *sm;
+ fifo_segment_t *fs;
u32 segment_index;
if (!rx_fifo || !tx_fifo)
segment_manager_segment_reader_unlock (sm);
}
+int
+segment_manager_grow_fifo (segment_manager_t * sm, svm_fifo_t * f, u32 size)
+{
+ fifo_segment_t *fs;
+ int rv;
+
+ fs = segment_manager_get_segment_w_lock (sm, f->segment_index);
+ rv = fifo_segment_grow_fifo (fs, f, size);
+ segment_manager_segment_reader_unlock (sm);
+
+ return rv;
+}
+
u32
segment_manager_evt_q_expected_size (u32 q_len)
{
fifo_segment_t *segment_manager_get_segment (segment_manager_t * sm,
u32 segment_index);
fifo_segment_t *segment_manager_get_segment_w_handle (u64 sh);
-fifo_segment_t *segment_manager_get_segment_w_lock (segment_manager_t *,
+fifo_segment_t *segment_manager_get_segment_w_lock (segment_manager_t * sm,
u32 segment_index);
int segment_manager_add_first_segment (segment_manager_t * sm,
u32 segment_size);
svm_fifo_t ** tx_fifo);
void segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo,
svm_fifo_t * tx_fifo);
+
+/**
+ * Grows fifo owned by segment manager
+ *
+ * @param sm segment manager that owns the fifo
+ * @param f fifo to be grown
+ * @param size amount of bytes to add to fifo
+ * @return 0 on success, negative number otherwise
+ */
+int segment_manager_grow_fifo (segment_manager_t * sm, svm_fifo_t * f,
+ u32 size);
u8 segment_manager_has_fifos (segment_manager_t * sm);
svm_msg_q_t *segment_manager_alloc_queue (fifo_segment_t * fs,
void
rb_tree_free_nodes (rb_tree_t * rt)
{
- rb_node_t *n;
- pool_flush (n, rt->nodes,;);
+ pool_free (rt->nodes);
+ rt->root = RBTREE_TNIL_INDEX;
}
void
Code Review / vpp.git / commitdiff