+CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t * f,
+ svm_fifo_chunk_t * c, u32 tail_idx, const u8 * src, u32 len,
+ svm_fifo_chunk_t ** last)
+{
+ u32 n_chunk;
+
+ ASSERT (tail_idx >= c->start_byte && tail_idx < c->start_byte + c->length);
+
+ tail_idx -= c->start_byte;
+ n_chunk = c->length - tail_idx;
+ if (n_chunk <= len)
+ {
+ u32 to_copy = len;
+ clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
+ c = c->next;
+ while ((to_copy -= n_chunk))
+ {
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
+ c = c->length <= to_copy ? c->next : c;
+ }
+ if (*last)
+ *last = c;
+ }
+ else
+ {
+ clib_memcpy_fast (&c->data[tail_idx], src, len);
+ }
+}
+
+CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t * f,
+ svm_fifo_chunk_t * c, u32 head_idx, u8 * dst, u32 len,
+ svm_fifo_chunk_t ** last)
+{
+ u32 n_chunk;
+
+ ASSERT (head_idx >= c->start_byte && head_idx < c->start_byte + c->length);
+
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
+ if (n_chunk <= len)
+ {
+ u32 to_copy = len;
+ clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+ c = c->next;
+ while ((to_copy -= n_chunk))
+ {
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
+ c = c->length <= to_copy ? c->next : c;
+ }
+ if (*last)
+ *last = c;
+ }
+ else
+ {
+ clib_memcpy_fast (dst, &c->data[head_idx], len);
+ }
+}
+
+#ifndef CLIB_MARCH_VARIANT
+
+static inline void
+svm_fifo_copy_to_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 tail_idx,
+ const u8 * src, u32 len, svm_fifo_chunk_t ** last)
+{
+ CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len,
+ last);
+}
+
+static inline void
+svm_fifo_copy_from_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 head_idx,
+ u8 * dst, u32 len, svm_fifo_chunk_t ** last)
+{
+ CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len,
+ last);
+}
+
+static inline u8
+position_lt (svm_fifo_t * f, u32 a, u32 b, u32 tail)
+{
+ return (ooo_segment_distance_from_tail (f, a, tail)
+ < ooo_segment_distance_from_tail (f, b, tail));
+}
+
+static inline u8
+position_leq (svm_fifo_t * f, u32 a, u32 b, u32 tail)
+{
+ return (ooo_segment_distance_from_tail (f, a, tail)
+ <= ooo_segment_distance_from_tail (f, b, tail));
+}
+
+static inline u8
+position_gt (svm_fifo_t * f, u32 a, u32 b, u32 tail)
+{
+ return (ooo_segment_distance_from_tail (f, a, tail)
+ > ooo_segment_distance_from_tail (f, b, tail));
+}
+
+static inline u32
+position_diff (svm_fifo_t * f, u32 posa, u32 posb, u32 tail)
+{
+ return ooo_segment_distance_from_tail (f, posa, tail)
+ - ooo_segment_distance_from_tail (f, posb, tail);
+}
+
+static inline u32
+ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s)
+{
+ return s->start + s->length;
+}
+
+u8 *
+format_ooo_segment (u8 * s, va_list * args)
+{
+ svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
+ ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
+ u32 normalized_start = (seg->start + f->nitems - f->tail) % f->size;
+ s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
+ (normalized_start + seg->length) % f->size, seg->length,
+ seg->next, seg->prev);
+ return s;
+}
+
+u8 *
+svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
+{
+#if SVM_FIFO_TRACE
+ svm_fifo_trace_elem_t *seg = 0;
+ int i = 0;
+
+ if (f->trace)
+ {
+ vec_foreach (seg, f->trace)
+ {
+ s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
+ i++;
+ if (i % 5 == 0)
+ s = format (s, "\n");
+ }
+ s = format (s, "\n");
+ }
+ return s;
+#else
+ return 0;
+#endif
+}
+
+u8 *
+svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
+{
+ int i, trace_len;
+ u8 *data = 0;
+ svm_fifo_trace_elem_t *trace;
+ u32 offset;
+ svm_fifo_t *dummy_fifo;
+
+ if (!f)
+ return s;
+
+#if SVM_FIFO_TRACE
+ trace = f->trace;
+ trace_len = vec_len (trace);
+#else
+ trace = 0;
+ trace_len = 0;
+#endif
+
+ dummy_fifo = svm_fifo_create (f->size);
+ clib_memset (f->head_chunk->data, 0xFF, f->nitems);
+ vec_validate (data, f->nitems);
+ for (i = 0; i < vec_len (data); i++)
+ data[i] = i;
+
+ for (i = 0; i < trace_len; i++)
+ {
+ offset = trace[i].offset;
+ if (trace[i].action == 1)
+ {
+ if (verbose)
+ s = format (s, "adding [%u, %u]:", trace[i].offset,
+ (trace[i].offset + trace[i].len) % dummy_fifo->size);
+ svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
+ trace[i].len, &data[offset]);
+ }
+ else if (trace[i].action == 2)
+ {
+ if (verbose)
+ s = format (s, "adding [%u, %u]:", 0, trace[i].len);
+ svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
+ }
+ else if (!no_read)
+ {
+ if (verbose)
+ s = format (s, "read: %u", trace[i].len);
+ svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
+ }
+ if (verbose)
+ s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
+ }
+
+ s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
+
+ return s;
+}
+
+u8 *
+format_ooo_list (u8 * s, va_list * args)
+{
+ svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
+ u32 indent = va_arg (*args, u32);
+ u32 ooo_segment_index = f->ooos_list_head;
+ ooo_segment_t *seg;
+
+ while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
+ {
+ seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
+ s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
+ f, seg);
+ ooo_segment_index = seg->next;
+ }
+
+ return s;
+}
+
+u8 *
+format_svm_fifo (u8 * s, va_list * args)
+{
+ svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
+ int verbose = va_arg (*args, int);
+ u32 indent;
+
+ if (!s)
+ return s;
+
+ indent = format_get_indent (s);
+ s = format (s, "cursize %u nitems %u has_event %d\n",
+ svm_fifo_max_dequeue (f), f->nitems, f->has_event);
+ s = format (s, "%Uhead %u tail %u segment manager %u\n", format_white_space,
+ indent, (f->head % f->size), (f->tail % f->size),
+ f->segment_manager);
+
+ if (verbose > 1)
+ s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
+ format_white_space, indent, f->master_session_index,
+ f->master_thread_index, f->client_session_index,
+ f->client_thread_index);
+
+ if (verbose)
+ {
+ s = format (s, "%Uooo pool %d active elts newest %u\n",
+ format_white_space, indent, pool_elts (f->ooo_segments),
+ f->ooos_newest);
+ if (svm_fifo_has_ooo_data (f))
+ s = format (s, " %U", format_ooo_list, f, indent, verbose);
+ }
+ return s;
+}
+
+void
+svm_fifo_init (svm_fifo_t * f, u32 size)
+{
+ f->size = size;
+ /*
+ * usable size of the fifo set to rounded_data_size - 1
+ * to differentiate between free fifo and empty fifo.
+ */
+ f->nitems = f->size - 1;
+ f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
+ f->refcnt = 1;
+ f->default_chunk.start_byte = 0;
+ f->default_chunk.length = f->size;
+ f->default_chunk.next = f->start_chunk = f->end_chunk = &f->default_chunk;
+ f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
+}
+
+/**
+ * Creates a fifo in the current heap. Fails vs blow up the process
+ */