-u8 *
-format_ooo_segment (u8 * s, va_list * args)
-{
- svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
- ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
- u32 normalized_start = (seg->start + f->nitems - f->tail) % f->size;
- s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
- (normalized_start + seg->length) % f->size, seg->length,
- seg->next, seg->prev);
- return s;
-}
-
-u8 *
-svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
-{
-#if SVM_FIFO_TRACE
- svm_fifo_trace_elem_t *seg = 0;
- int i = 0;
-
- if (f->trace)
- {
- vec_foreach (seg, f->trace)
- {
- s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
- i++;
- if (i % 5 == 0)
- s = format (s, "\n");
- }
- s = format (s, "\n");
- }
- return s;
-#else
- return 0;
-#endif
-}
-
-u8 *
-svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
-{
- int i, trace_len;
- u8 *data = 0;
- svm_fifo_trace_elem_t *trace;
- u32 offset;
- svm_fifo_t *dummy_fifo;
-
- if (!f)
- return s;
-
-#if SVM_FIFO_TRACE
- trace = f->trace;
- trace_len = vec_len (trace);
-#else
- trace = 0;
- trace_len = 0;
-#endif
-
- dummy_fifo = svm_fifo_create (f->size);
- clib_memset (f->head_chunk->data, 0xFF, f->nitems);
- vec_validate (data, f->nitems);
- for (i = 0; i < vec_len (data); i++)
- data[i] = i;
-
- for (i = 0; i < trace_len; i++)
- {
- offset = trace[i].offset;
- if (trace[i].action == 1)
- {
- if (verbose)
- s = format (s, "adding [%u, %u]:", trace[i].offset,
- (trace[i].offset + trace[i].len) % dummy_fifo->size);
- svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
- trace[i].len, &data[offset]);
- }
- else if (trace[i].action == 2)
- {
- if (verbose)
- s = format (s, "adding [%u, %u]:", 0, trace[i].len);
- svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
- }
- else if (!no_read)
- {
- if (verbose)
- s = format (s, "read: %u", trace[i].len);
- svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
- }
- if (verbose)
- s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
- }
-
- s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
-
- return s;
-}
-
-u8 *
-format_ooo_list (u8 * s, va_list * args)
-{
- svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
- u32 indent = va_arg (*args, u32);
- u32 ooo_segment_index = f->ooos_list_head;
- ooo_segment_t *seg;
-
- while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
- {
- seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
- s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
- f, seg);
- ooo_segment_index = seg->next;
- }
-
- return s;
-}
-
-u8 *
-format_svm_fifo (u8 * s, va_list * args)
-{
- svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
- int verbose = va_arg (*args, int);
- u32 indent;
-
- if (!s)
- return s;
-
- indent = format_get_indent (s);
- s = format (s, "cursize %u nitems %u has_event %d\n",
- svm_fifo_max_dequeue (f), f->nitems, f->has_event);
- s = format (s, "%Uhead %u tail %u segment manager %u\n", format_white_space,
- indent, (f->head % f->size), (f->tail % f->size),
- f->segment_manager);
-
- if (verbose > 1)
- s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
- format_white_space, indent, f->master_session_index,
- f->master_thread_index, f->client_session_index,
- f->client_thread_index);
-
- if (verbose)
- {
- s = format (s, "%Uooo pool %d active elts newest %u\n",
- format_white_space, indent, pool_elts (f->ooo_segments),
- f->ooos_newest);
- if (svm_fifo_has_ooo_data (f))
- s = format (s, " %U", format_ooo_list, f, indent, verbose);
- }
- return s;
-}
-
-void
-svm_fifo_init (svm_fifo_t * f, u32 size)
-{
- f->size = size;
- /*
- * usable size of the fifo set to rounded_data_size - 1
- * to differentiate between free fifo and empty fifo.
- */
- f->nitems = f->size - 1;
- f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
- f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
- f->segment_index = SVM_FIFO_INVALID_INDEX;
- f->refcnt = 1;
- f->default_chunk.start_byte = 0;
- f->default_chunk.length = f->size;
- f->default_chunk.next = f->start_chunk = f->end_chunk = &f->default_chunk;
- f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
-}
-
-/**
- * Creates a fifo in the current heap. Fails vs blow up the process
- */
-svm_fifo_t *
-svm_fifo_create (u32 data_size_in_bytes)
-{
- svm_fifo_t *f;
- u32 rounded_data_size;
-
- /* always round fifo data size to the next highest power-of-two */
- rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
- f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
- CLIB_CACHE_LINE_BYTES);
- if (f == 0)
- return 0;
-
- clib_memset (f, 0, sizeof (*f));
- svm_fifo_init (f, data_size_in_bytes);
- return f;
-}
-
-/**
- * Creates a fifo chunk in the current heap
- */
-svm_fifo_chunk_t *
-svm_fifo_chunk_alloc (u32 size)
-{
- svm_fifo_chunk_t *c;
- u32 rounded_size;
-
- /* round chunk size to the next highest power-of-two */
- rounded_size = (1 << (max_log2 (size)));
- c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size,
- CLIB_CACHE_LINE_BYTES);
- if (c == 0)
- return 0;
-
- clib_memset (c, 0, sizeof (*c));
- c->length = rounded_size;
- return c;
-}
-
-static inline void
-svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c)
-{
- svm_fifo_chunk_t *prev;
- u32 add_bytes = 0;
-
- if (!c)
- return;
-
- f->end_chunk->next = c;
- while (c)
- {
- add_bytes += c->length;
- prev = c;
- c = c->next;
- }
- f->end_chunk = prev;
- prev->next = f->start_chunk;
- f->size += add_bytes;
- f->nitems = f->size - 1;
- f->new_chunks = 0;
-}
-
-static void
-svm_fifo_try_size_update (svm_fifo_t * f, u32 new_head)
-{
- if (new_head % f->size > f->tail % f->size)
- return;
-
- svm_fifo_size_update (f, f->new_chunks);
- f->flags &= ~SVM_FIFO_F_SIZE_UPDATE;
-}
-
-void
-svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c)
-{
- svm_fifo_chunk_t *cur, *prev;
-
- /* Initialize rbtree if needed and add default chunk to it */
- if (!(f->flags & SVM_FIFO_F_MULTI_CHUNK))
- {
- rb_tree_init (&f->chunk_lookup);
- rb_tree_add2 (&f->chunk_lookup, 0, pointer_to_uword (f->start_chunk));
- f->flags |= SVM_FIFO_F_MULTI_CHUNK;
- }
-
- /* Initialize chunks and add to lookup rbtree. Expectation is that this is
- * called with the heap where the rbtree's pool is pushed. */
- cur = c;
- if (f->new_chunks)
- {
- prev = f->new_chunks;
- while (prev->next)
- prev = prev->next;
- prev->next = c;
- }
- else
- prev = f->end_chunk;
-
- while (cur)
- {
- cur->start_byte = prev->start_byte + prev->length;
- rb_tree_add2 (&f->chunk_lookup, cur->start_byte,
- pointer_to_uword (cur));
- prev = cur;
- cur = cur->next;
- }
-
- /* If fifo is not wrapped, update the size now */
- if (!svm_fifo_is_wrapped (f))
- {
- ASSERT (!f->new_chunks);
- svm_fifo_size_update (f, c);
- return;
- }
-
- /* Postpone size update */
- if (!f->new_chunks)
- {
- f->new_chunks = c;
- f->flags |= SVM_FIFO_F_SIZE_UPDATE;
- }
-}
-
-static inline u8
-svm_fifo_chunk_includes_pos (svm_fifo_chunk_t * c, u32 pos)
-{
- return (pos >= c->start_byte && pos < c->start_byte + c->length);
-}
-
-/**
- * Find chunk for given byte position
- *
- * @param f fifo
- * @param pos normalized position in fifo
- *
- * @return chunk that includes given position or 0
- */
-static svm_fifo_chunk_t *
-svm_fifo_find_chunk (svm_fifo_t * f, u32 pos)
-{
- rb_tree_t *rt = &f->chunk_lookup;
- rb_node_t *cur, *prev;
- svm_fifo_chunk_t *c;
-
- cur = rb_node (rt, rt->root);
- while (pos != cur->key)
- {
- prev = cur;
- if (pos < cur->key)
- cur = rb_node_left (rt, cur);
- else
- cur = rb_node_right (rt, cur);
-
- if (rb_node_is_tnil (rt, cur))
- {
- /* Hit tnil as a left child. Find predecessor */
- if (pos < prev->key)
- {
- cur = rb_tree_predecessor (rt, prev);
- c = uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
- if (svm_fifo_chunk_includes_pos (c, pos))
- return c;
- return 0;
- }
- /* Hit tnil as a right child. Check if this is the one, otherwise
- * search for successor */
- c = uword_to_pointer (prev->opaque, svm_fifo_chunk_t *);
- if (svm_fifo_chunk_includes_pos (c, pos))
- return c;
-
- cur = rb_tree_successor (rt, prev);
- c = uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
- if (svm_fifo_chunk_includes_pos (c, pos))
- return c;
- return 0;
- }
- }
-
- if (!rb_node_is_tnil (rt, cur))
- return uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
- return 0;
-}
-
-void
-svm_fifo_free_chunk_lookup (svm_fifo_t * f)
-{
- rb_tree_free_nodes (&f->chunk_lookup);
-}
-