2 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3 * Copyright (c) 2019 Arm Limited
4 * Copyright (c) 2010-2017 Intel Corporation and/or its affiliates.
5 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
6 * Inspired from DPDK rte_ring.h (SPSC only) (derived from freebsd bufring.h).
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at:
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 #include <svm/svm_fifo.h>
21 #include <vppinfra/cpu.h>
23 CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t * f,
24 svm_fifo_chunk_t * c, u32 tail_idx, const u8 * src, u32 len,
25 svm_fifo_chunk_t ** last)
29 ASSERT (tail_idx >= c->start_byte && tail_idx < c->start_byte + c->length);
31 tail_idx -= c->start_byte;
32 n_chunk = c->length - tail_idx;
36 clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
38 while ((to_copy -= n_chunk))
40 n_chunk = clib_min (c->length, to_copy);
41 clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
42 c = c->length <= to_copy ? c->next : c;
49 clib_memcpy_fast (&c->data[tail_idx], src, len);
53 CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t * f,
54 svm_fifo_chunk_t * c, u32 head_idx, u8 * dst, u32 len,
55 svm_fifo_chunk_t ** last)
59 ASSERT (head_idx >= c->start_byte && head_idx < c->start_byte + c->length);
61 head_idx -= c->start_byte;
62 n_chunk = c->length - head_idx;
66 clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
68 while ((to_copy -= n_chunk))
70 n_chunk = clib_min (c->length, to_copy);
71 clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
72 c = c->length <= to_copy ? c->next : c;
79 clib_memcpy_fast (dst, &c->data[head_idx], len);
83 #ifndef CLIB_MARCH_VARIANT
86 svm_fifo_copy_to_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 tail_idx,
87 const u8 * src, u32 len, svm_fifo_chunk_t ** last)
89 CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len,
94 svm_fifo_copy_from_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 head_idx,
95 u8 * dst, u32 len, svm_fifo_chunk_t ** last)
97 CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len,
102 position_lt (svm_fifo_t * f, u32 a, u32 b, u32 tail)
104 return (ooo_segment_distance_from_tail (f, a, tail)
105 < ooo_segment_distance_from_tail (f, b, tail));
109 position_leq (svm_fifo_t * f, u32 a, u32 b, u32 tail)
111 return (ooo_segment_distance_from_tail (f, a, tail)
112 <= ooo_segment_distance_from_tail (f, b, tail));
116 position_gt (svm_fifo_t * f, u32 a, u32 b, u32 tail)
118 return (ooo_segment_distance_from_tail (f, a, tail)
119 > ooo_segment_distance_from_tail (f, b, tail));
123 position_diff (svm_fifo_t * f, u32 posa, u32 posb, u32 tail)
125 return ooo_segment_distance_from_tail (f, posa, tail)
126 - ooo_segment_distance_from_tail (f, posb, tail);
130 ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s)
132 return s->start + s->length;
136 format_ooo_segment (u8 * s, va_list * args)
138 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
139 ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
140 u32 normalized_start = (seg->start + f->nitems - f->tail) % f->size;
141 s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
142 (normalized_start + seg->length) % f->size, seg->length,
143 seg->next, seg->prev);
148 svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
151 svm_fifo_trace_elem_t *seg = 0;
156 vec_foreach (seg, f->trace)
158 s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
161 s = format (s, "\n");
163 s = format (s, "\n");
172 svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
176 svm_fifo_trace_elem_t *trace;
178 svm_fifo_t *dummy_fifo;
185 trace_len = vec_len (trace);
191 dummy_fifo = svm_fifo_create (f->size);
192 clib_memset (f->head_chunk->data, 0xFF, f->nitems);
193 vec_validate (data, f->nitems);
194 for (i = 0; i < vec_len (data); i++)
197 for (i = 0; i < trace_len; i++)
199 offset = trace[i].offset;
200 if (trace[i].action == 1)
203 s = format (s, "adding [%u, %u]:", trace[i].offset,
204 (trace[i].offset + trace[i].len) % dummy_fifo->size);
205 svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
206 trace[i].len, &data[offset]);
208 else if (trace[i].action == 2)
211 s = format (s, "adding [%u, %u]:", 0, trace[i].len);
212 svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
217 s = format (s, "read: %u", trace[i].len);
218 svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
221 s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
224 s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
230 format_ooo_list (u8 * s, va_list * args)
232 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
233 u32 indent = va_arg (*args, u32);
234 u32 ooo_segment_index = f->ooos_list_head;
237 while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
239 seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
240 s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
242 ooo_segment_index = seg->next;
249 format_svm_fifo (u8 * s, va_list * args)
251 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
252 int verbose = va_arg (*args, int);
258 indent = format_get_indent (s);
259 s = format (s, "cursize %u nitems %u has_event %d\n",
260 svm_fifo_max_dequeue (f), f->nitems, f->has_event);
261 s = format (s, "%Uhead %u tail %u segment manager %u\n", format_white_space,
262 indent, (f->head % f->size), (f->tail % f->size),
266 s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
267 format_white_space, indent, f->master_session_index,
268 f->master_thread_index, f->client_session_index,
269 f->client_thread_index);
273 s = format (s, "%Uooo pool %d active elts newest %u\n",
274 format_white_space, indent, pool_elts (f->ooo_segments),
276 if (svm_fifo_has_ooo_data (f))
277 s = format (s, " %U", format_ooo_list, f, indent, verbose);
283 svm_fifo_init (svm_fifo_t * f, u32 size)
287 * usable size of the fifo set to rounded_data_size - 1
288 * to differentiate between free fifo and empty fifo.
290 f->nitems = f->size - 1;
291 f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
292 f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
293 f->segment_index = SVM_FIFO_INVALID_INDEX;
295 f->default_chunk.start_byte = 0;
296 f->default_chunk.length = f->size;
297 f->default_chunk.next = f->start_chunk = f->end_chunk = &f->default_chunk;
298 f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
302 * Creates a fifo in the current heap. Fails vs blow up the process
305 svm_fifo_create (u32 data_size_in_bytes)
308 u32 rounded_data_size;
310 /* always round fifo data size to the next highest power-of-two */
311 rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
312 f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
313 CLIB_CACHE_LINE_BYTES);
317 clib_memset (f, 0, sizeof (*f));
318 svm_fifo_init (f, data_size_in_bytes);
323 * Creates a fifo chunk in the current heap
326 svm_fifo_chunk_alloc (u32 size)
331 /* round chunk size to the next highest power-of-two */
332 rounded_size = (1 << (max_log2 (size)));
333 c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size,
334 CLIB_CACHE_LINE_BYTES);
338 clib_memset (c, 0, sizeof (*c));
339 c->length = rounded_size;
344 svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c)
346 svm_fifo_chunk_t *prev;
352 f->end_chunk->next = c;
355 add_bytes += c->length;
360 prev->next = f->start_chunk;
361 f->size += add_bytes;
362 f->nitems = f->size - 1;
367 svm_fifo_try_size_update (svm_fifo_t * f, u32 new_head)
369 if (new_head % f->size > f->tail % f->size)
372 svm_fifo_size_update (f, f->new_chunks);
373 f->flags &= ~SVM_FIFO_F_SIZE_UPDATE;
377 svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c)
379 svm_fifo_chunk_t *cur, *prev;
381 /* Initialize rbtree if needed and add default chunk to it */
382 if (!(f->flags & SVM_FIFO_F_MULTI_CHUNK))
384 rb_tree_init (&f->chunk_lookup);
385 rb_tree_add2 (&f->chunk_lookup, 0, pointer_to_uword (f->start_chunk));
386 f->flags |= SVM_FIFO_F_MULTI_CHUNK;
389 /* Initialize chunks and add to lookup rbtree. Expectation is that this is
390 * called with the heap where the rbtree's pool is pushed. */
394 prev = f->new_chunks;
404 cur->start_byte = prev->start_byte + prev->length;
405 rb_tree_add2 (&f->chunk_lookup, cur->start_byte,
406 pointer_to_uword (cur));
411 /* If fifo is not wrapped, update the size now */
412 if (!svm_fifo_is_wrapped (f))
414 ASSERT (!f->new_chunks);
415 svm_fifo_size_update (f, c);
419 /* Postpone size update */
423 f->flags |= SVM_FIFO_F_SIZE_UPDATE;
428 svm_fifo_chunk_includes_pos (svm_fifo_chunk_t * c, u32 pos)
430 return (pos >= c->start_byte && pos < c->start_byte + c->length);
434 * Find chunk for given byte position
437 * @param pos normalized position in fifo
439 * @return chunk that includes given position or 0
441 static svm_fifo_chunk_t *
442 svm_fifo_find_chunk (svm_fifo_t * f, u32 pos)
444 rb_tree_t *rt = &f->chunk_lookup;
445 rb_node_t *cur, *prev;
448 cur = rb_node (rt, rt->root);
449 while (pos != cur->key)
453 cur = rb_node_left (rt, cur);
455 cur = rb_node_right (rt, cur);
457 if (rb_node_is_tnil (rt, cur))
459 /* Hit tnil as a left child. Find predecessor */
462 cur = rb_tree_predecessor (rt, prev);
463 c = uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
464 if (svm_fifo_chunk_includes_pos (c, pos))
468 /* Hit tnil as a right child. Check if this is the one, otherwise
469 * search for successor */
470 c = uword_to_pointer (prev->opaque, svm_fifo_chunk_t *);
471 if (svm_fifo_chunk_includes_pos (c, pos))
474 cur = rb_tree_successor (rt, prev);
475 c = uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
476 if (svm_fifo_chunk_includes_pos (c, pos))
482 if (!rb_node_is_tnil (rt, cur))
483 return uword_to_pointer (cur->opaque, svm_fifo_chunk_t *);
488 svm_fifo_free_chunk_lookup (svm_fifo_t * f)
490 rb_tree_free_nodes (&f->chunk_lookup);
494 svm_fifo_free_ooo_data (svm_fifo_t * f)
496 pool_free (f->ooo_segments);
500 svm_fifo_free (svm_fifo_t * f)
502 ASSERT (f->refcnt > 0);
504 if (--f->refcnt == 0)
506 /* ooo data is not allocated on segment heap */
507 svm_fifo_free_chunk_lookup (f);
512 always_inline ooo_segment_t *
513 ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
517 pool_get (f->ooo_segments, s);
522 s->prev = s->next = OOO_SEGMENT_INVALID_INDEX;
528 ooo_segment_del (svm_fifo_t * f, u32 index)
530 ooo_segment_t *cur, *prev = 0, *next = 0;
531 cur = pool_elt_at_index (f->ooo_segments, index);
533 if (cur->next != OOO_SEGMENT_INVALID_INDEX)
535 next = pool_elt_at_index (f->ooo_segments, cur->next);
536 next->prev = cur->prev;
539 if (cur->prev != OOO_SEGMENT_INVALID_INDEX)
541 prev = pool_elt_at_index (f->ooo_segments, cur->prev);
542 prev->next = cur->next;
546 f->ooos_list_head = cur->next;
549 pool_put (f->ooo_segments, cur);
553 * Add segment to fifo's out-of-order segment list. Takes care of merging
554 * adjacent segments and removing overlapping ones.
557 ooo_segment_add (svm_fifo_t * f, u32 offset, u32 head, u32 tail, u32 length)
559 ooo_segment_t *s, *new_s, *prev, *next, *it;
560 u32 new_index, s_end_pos, s_index;
561 u32 offset_pos, offset_end_pos;
563 ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail)
566 offset_pos = tail + offset;
567 offset_end_pos = tail + offset + length;
569 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
571 if (f->ooos_list_head == OOO_SEGMENT_INVALID_INDEX)
573 s = ooo_segment_new (f, offset_pos, length);
574 f->ooos_list_head = s - f->ooo_segments;
575 f->ooos_newest = f->ooos_list_head;
579 /* Find first segment that starts after new segment */
580 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
581 while (s->next != OOO_SEGMENT_INVALID_INDEX
582 && position_lt (f, s->start, offset_pos, tail))
583 s = pool_elt_at_index (f->ooo_segments, s->next);
585 /* If we have a previous and we overlap it, use it as starting point */
586 prev = ooo_segment_get_prev (f, s);
588 && position_leq (f, offset_pos, ooo_segment_end_pos (f, prev), tail))
591 s_end_pos = ooo_segment_end_pos (f, s);
593 /* Since we have previous, offset start position cannot be smaller
594 * than prev->start. Check tail */
595 ASSERT (position_lt (f, s->start, offset_pos, tail));
599 s_index = s - f->ooo_segments;
600 s_end_pos = ooo_segment_end_pos (f, s);
602 /* No overlap, add before current segment */
603 if (position_lt (f, offset_end_pos, s->start, tail))
605 new_s = ooo_segment_new (f, offset_pos, length);
606 new_index = new_s - f->ooo_segments;
608 /* Pool might've moved, get segment again */
609 s = pool_elt_at_index (f->ooo_segments, s_index);
610 if (s->prev != OOO_SEGMENT_INVALID_INDEX)
612 new_s->prev = s->prev;
613 prev = pool_elt_at_index (f->ooo_segments, new_s->prev);
614 prev->next = new_index;
619 f->ooos_list_head = new_index;
622 new_s->next = s_index;
624 f->ooos_newest = new_index;
627 /* No overlap, add after current segment */
628 else if (position_gt (f, offset_pos, s_end_pos, tail))
630 new_s = ooo_segment_new (f, offset_pos, length);
631 new_index = new_s - f->ooo_segments;
633 /* Pool might've moved, get segment again */
634 s = pool_elt_at_index (f->ooo_segments, s_index);
636 /* Needs to be last */
637 ASSERT (s->next == OOO_SEGMENT_INVALID_INDEX);
639 new_s->prev = s_index;
641 f->ooos_newest = new_index;
651 if (position_lt (f, offset_pos, s->start, tail))
653 s->start = offset_pos;
654 s->length = position_diff (f, s_end_pos, s->start, tail);
655 f->ooos_newest = s - f->ooo_segments;
660 /* Overlapping tail */
661 if (position_gt (f, offset_end_pos, s_end_pos, tail))
663 s->length = position_diff (f, offset_end_pos, s->start, tail);
665 /* Remove the completely overlapped segments in the tail */
666 it = ooo_segment_next (f, s);
667 while (it && position_leq (f, ooo_segment_end_pos (f, it),
668 offset_end_pos, tail))
670 next = ooo_segment_next (f, it);
671 ooo_segment_del (f, it - f->ooo_segments);
675 /* If partial overlap with last, merge */
676 if (it && position_leq (f, it->start, offset_end_pos, tail))
678 s->length = position_diff (f, ooo_segment_end_pos (f, it),
680 ooo_segment_del (f, it - f->ooo_segments);
682 f->ooos_newest = s - f->ooo_segments;
687 * Removes segments that can now be enqueued because the fifo's tail has
688 * advanced. Returns the number of bytes added to tail.
691 ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued, u32 * tail)
694 u32 index, bytes = 0;
697 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
698 diff = ooo_segment_distance_to_tail (f, s->start, *tail);
700 ASSERT (diff != n_bytes_enqueued);
702 if (diff > n_bytes_enqueued)
705 /* If last tail update overlaps one/multiple ooo segments, remove them */
706 while (0 <= diff && diff < n_bytes_enqueued)
708 index = s - f->ooo_segments;
710 /* Segment end is beyond the tail. Advance tail and remove segment */
711 if (s->length > diff)
713 bytes = s->length - diff;
714 *tail = *tail + bytes;
715 ooo_segment_del (f, index);
719 /* If we have next go on */
720 if (s->next != OOO_SEGMENT_INVALID_INDEX)
722 s = pool_elt_at_index (f->ooo_segments, s->next);
723 diff = ooo_segment_distance_to_tail (f, s->start, *tail);
724 ooo_segment_del (f, index);
729 ooo_segment_del (f, index);
734 ASSERT (bytes <= f->nitems);
739 svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
742 u32 head, tail, head_idx;
745 ASSERT (len <= f->nitems);
747 f_load_head_tail_cons (f, &head, &tail);
749 head_idx = head % f->size;
750 head_idx -= c->start_byte;
751 n_chunk = c->length - head_idx;
753 clib_memcpy_fast (&c->data[head_idx], data, len);
756 clib_memcpy_fast (&c->data[head_idx], data, n_chunk);
757 clib_memcpy_fast (&c->next->data[0], data + n_chunk, len - n_chunk);
762 svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 len, const u8 * src)
764 u32 tail, head, free_count;
766 f_load_head_tail_prod (f, &head, &tail);
768 /* free space in fifo can only increase during enqueue: SPSC */
769 free_count = f_free_count (f, head, tail);
771 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
773 if (PREDICT_FALSE (free_count == 0))
774 return SVM_FIFO_FULL;
776 /* number of bytes we're going to copy */
777 len = clib_min (free_count, len);
779 svm_fifo_copy_to_chunk (f, f->tail_chunk, tail % f->size, src, len,
783 svm_fifo_trace_add (f, head, n_total, 2);
785 /* collect out-of-order segments */
786 if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
787 len += ooo_segment_try_collect (f, len, &tail);
789 /* store-rel: producer owned index (paired with load-acq in consumer) */
790 clib_atomic_store_rel_n (&f->tail, tail);
796 * Enqueue a future segment.
798 * Two choices: either copies the entire segment, or copies nothing
799 * Returns 0 of the entire segment was copied
800 * Returns -1 if none of the segment was copied due to lack of space
803 svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
805 u32 tail, head, free_count, tail_idx;
807 f_load_head_tail_prod (f, &head, &tail);
809 /* free space in fifo can only increase during enqueue: SPSC */
810 free_count = f_free_count (f, head, tail);
812 /* will this request fit? */
813 if ((len + offset) > free_count)
816 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
818 svm_fifo_trace_add (f, offset, len, 1);
820 ooo_segment_add (f, offset, head, tail, len);
822 tail_idx = (tail % f->size + offset) % f->size;
824 if (!svm_fifo_chunk_includes_pos (f->ooo_enq, tail_idx))
825 f->ooo_enq = svm_fifo_find_chunk (f, tail_idx);
827 svm_fifo_copy_to_chunk (f, f->ooo_enq, tail_idx, src, len, &f->ooo_enq);
833 svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 len, u8 * dst)
835 u32 tail, head, cursize;
837 f_load_head_tail_cons (f, &head, &tail);
839 /* current size of fifo can only increase during dequeue: SPSC */
840 cursize = f_cursize (f, head, tail);
842 if (PREDICT_FALSE (cursize == 0))
843 return -2; /* nothing in the fifo */
845 len = clib_min (cursize, len);
847 svm_fifo_copy_from_chunk (f, f->head_chunk, head % f->size, dst, len,
851 if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SIZE_UPDATE))
852 svm_fifo_try_size_update (f, head);
854 /* store-rel: consumer owned index (paired with load-acq in producer) */
855 clib_atomic_store_rel_n (&f->head, head);
861 svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst)
863 u32 tail, head, cursize, head_idx;
865 f_load_head_tail_cons (f, &head, &tail);
867 /* current size of fifo can only increase during peek: SPSC */
868 cursize = f_cursize (f, head, tail);
870 if (PREDICT_FALSE (cursize < offset))
871 return -2; /* nothing in the fifo */
873 len = clib_min (cursize - offset, len);
874 head_idx = (head % f->size + offset) % f->size;
875 if (!svm_fifo_chunk_includes_pos (f->ooo_deq, head_idx))
876 f->ooo_deq = svm_fifo_find_chunk (f, head_idx);
878 svm_fifo_copy_from_chunk (f, f->ooo_deq, head_idx, dst, len, &f->ooo_deq);
883 svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
885 u32 total_drop_bytes;
886 u32 tail, head, cursize;
888 f_load_head_tail_cons (f, &head, &tail);
890 /* number of bytes we're going to drop */
891 cursize = f_cursize (f, head, tail);
893 if (PREDICT_FALSE (cursize == 0))
894 return -2; /* nothing in the fifo */
896 svm_fifo_trace_add (f, tail, total_drop_bytes, 3);
898 /* number of bytes we're going to drop */
899 total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
902 head += total_drop_bytes;
904 ASSERT (cursize >= total_drop_bytes);
905 /* store-rel: consumer owned index (paired with load-acq in producer) */
906 clib_atomic_store_rel_n (&f->head, head);
908 return total_drop_bytes;
912 svm_fifo_dequeue_drop_all (svm_fifo_t * f)
914 /* consumer foreign index */
915 u32 tail = clib_atomic_load_acq_n (&f->tail);
916 /* store-rel: consumer owned index (paired with load-acq in producer) */
917 clib_atomic_store_rel_n (&f->head, tail);
921 svm_fifo_segments (svm_fifo_t * f, svm_fifo_seg_t * fs)
923 u32 cursize, head, tail, head_idx;
925 f_load_head_tail_cons (f, &head, &tail);
927 /* consumer function, cursize can only increase while we're working */
928 cursize = f_cursize (f, head, tail);
930 if (PREDICT_FALSE (cursize == 0))
931 return -2; /* nothing in the fifo */
933 head_idx = head % f->size;
937 fs[0].len = f->size - head_idx;
938 fs[0].data = f->head_chunk->data + head_idx;
939 fs[1].len = cursize - fs[0].len;
940 fs[1].data = f->head_chunk->data;
945 fs[0].data = f->head_chunk->data + head_idx;
953 svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_seg_t * fs)
957 /* consumer owned index */
959 head_idx = head % f->size;
961 ASSERT (fs[0].data == f->head_chunk->data + head_idx);
962 head += fs[0].len + fs[1].len;
963 /* store-rel: consumer owned index (paired with load-acq in producer) */
964 clib_atomic_store_rel_n (&f->head, head);
967 /* Assumption: no prod and cons are accessing either dest or src fifo */
969 svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf)
972 clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size);
974 f_load_head_tail_all_acq (sf, &head, &tail);
975 clib_atomic_store_rel_n (&df->head, head);
976 clib_atomic_store_rel_n (&df->tail, tail);
980 svm_fifo_number_ooo_segments (svm_fifo_t * f)
982 return pool_elts (f->ooo_segments);
986 svm_fifo_first_ooo_segment (svm_fifo_t * f)
988 return pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
992 * Set fifo pointers to requested offset
995 svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail)
997 clib_atomic_store_rel_n (&f->head, head);
998 clib_atomic_store_rel_n (&f->tail, tail);
999 if (f->flags & SVM_FIFO_F_MULTI_CHUNK)
1001 svm_fifo_chunk_t *c;
1002 c = svm_fifo_find_chunk (f, head % f->size);
1004 f->head_chunk = f->ooo_deq = c;
1005 c = svm_fifo_find_chunk (f, tail % f->size);
1007 f->tail_chunk = f->ooo_enq = c;
1012 svm_fifo_add_subscriber (svm_fifo_t * f, u8 subscriber)
1014 if (f->n_subscribers >= SVM_FIFO_MAX_EVT_SUBSCRIBERS)
1016 f->subscribers[f->n_subscribers++] = subscriber;
1020 svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber)
1024 for (i = 0; i < f->n_subscribers; i++)
1026 if (f->subscribers[i] != subscriber)
1028 f->subscribers[i] = f->subscribers[f->n_subscribers - 1];
1036 * fd.io coding-style-patch-verification: ON
1039 * eval: (c-set-style "gnu")