X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=aa523c6b55c79f0de9f069dae1d7ccbe61ec0678;hb=430634c457da5dd04f481da0118bab581ace732e;hp=0c258b4259c515b65a4204e5915d5084560b9a4e;hpb=2cba8533cc4444c4615903add9a8f20c7c87079c;p=vpp.git diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 0c258b4259c..aa523c6b55c 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -50,6 +50,8 @@ ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s) return (s->start + s->length) % f->nitems; } +#ifndef CLIB_MARCH_VARIANT + u8 * format_ooo_segment (u8 * s, va_list * args) { @@ -210,6 +212,7 @@ svm_fifo_create (u32 data_size_in_bytes) memset (f, 0, sizeof (*f)); f->nitems = data_size_in_bytes; f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX; f->refcnt = 1; return (f); } @@ -225,6 +228,7 @@ svm_fifo_free (svm_fifo_t * f) clib_mem_free (f); } } +#endif always_inline ooo_segment_t * ooo_segment_new (svm_fifo_t * f, u32 start, u32 length) @@ -451,9 +455,8 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued) return bytes; } -static int -svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -510,45 +513,20 @@ svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, /* Atomically increase the queue length */ ASSERT (cursize + total_copy_bytes <= nitems); - __sync_fetch_and_add (&f->cursize, total_copy_bytes); + clib_atomic_fetch_add (&f->cursize, total_copy_bytes); return (total_copy_bytes); } -#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \ - { return fn (f, max_bytes, copy_from_here);} - -static int -svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) -{ - return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here); -} - -foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE, - svm_fifo_enqueue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma); - +#ifndef CLIB_MARCH_VARIANT int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here); -#else - static int (*fp) (svm_fifo_t *, u32, const u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_from_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes, + copy_from_here); } +#endif /** * Enqueue a future segment. @@ -557,11 +535,8 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, * Returns 0 of the entire segment was copied * Returns -1 if none of the segment was copied due to lack of space */ -static int -svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, - u32 offset, - u32 required_bytes, - u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, + u32 offset, u32 required_bytes, u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, normalized_offset; @@ -609,14 +584,15 @@ svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, return (0); } +#ifndef CLIB_MARCH_VARIANT int -svm_fifo_enqueue_with_offset (svm_fifo_t * f, - u32 offset, - u32 required_bytes, u8 * copy_from_here) +svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes, + u8 * copy_from_here) { - return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes, - copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset, + required_bytes, + copy_from_here); } void @@ -633,9 +609,10 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk); } } +#endif -static int -svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, + u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -682,48 +659,23 @@ svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) ASSERT (f->head <= nitems); ASSERT (cursize >= total_copy_bytes); - __sync_fetch_and_sub (&f->cursize, total_copy_bytes); + clib_atomic_fetch_sub (&f->cursize, total_copy_bytes); return (total_copy_bytes); } -static int -svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) -{ - return svm_fifo_dequeue_internal (f, max_bytes, copy_here); -} - -#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE, - svm_fifo_dequeue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes, + copy_here); } +#endif -static int -svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, + u32 max_bytes, u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, real_head; @@ -760,32 +712,14 @@ svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, return total_copy_bytes; } -#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, relative_offset, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_peek_ma_multiarch_select (); - - return (*fp) (f, relative_offset, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes, + copy_here); } int @@ -823,7 +757,7 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes) ASSERT (f->head <= nitems); ASSERT (cursize >= total_drop_bytes); - __sync_fetch_and_sub (&f->cursize, total_drop_bytes); + clib_atomic_fetch_sub (&f->cursize, total_drop_bytes); return total_drop_bytes; } @@ -832,7 +766,7 @@ void svm_fifo_dequeue_drop_all (svm_fifo_t * f) { f->head = f->tail; - __sync_fetch_and_sub (&f->cursize, f->cursize); + clib_atomic_fetch_sub (&f->cursize, f->cursize); } int @@ -879,7 +813,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs) f->head = (f->head + fs[0].len) % f->nitems; total_drop_bytes = fs[0].len; } - __sync_fetch_and_sub (&f->cursize, total_drop_bytes); + clib_atomic_fetch_sub (&f->cursize, total_drop_bytes); } u32 @@ -903,6 +837,7 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer) f->head = f->tail = pointer % f->nitems; } +#endif /* * fd.io coding-style-patch-verification: ON *