X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=4eae0a1c7e68b38c6a64ca267e157841e39003c9;hb=326b81e30e63a8296df51d85e6514356cd737225;hp=018827e0343681e74bfa5d5804f43e4d5f5af4a9;hpb=54693d23307ce8944a4d97379efd3bd4dcf0485c;p=vpp.git diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 018827e0343..4eae0a1c7e6 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -50,6 +50,8 @@ ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s) return (s->start + s->length) % f->nitems; } +#ifndef CLIB_MARCH_VARIANT + u8 * format_ooo_segment (u8 * s, va_list * args) { @@ -210,6 +212,7 @@ svm_fifo_create (u32 data_size_in_bytes) memset (f, 0, sizeof (*f)); f->nitems = data_size_in_bytes; f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX; f->refcnt = 1; return (f); } @@ -225,6 +228,7 @@ svm_fifo_free (svm_fifo_t * f) clib_mem_free (f); } } +#endif always_inline ooo_segment_t * ooo_segment_new (svm_fifo_t * f, u32 start, u32 length) @@ -451,9 +455,8 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued) return bytes; } -static int -svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -515,40 +518,15 @@ svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, return (total_copy_bytes); } -#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \ - { return fn (f, max_bytes, copy_from_here);} - -static int -svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) -{ - return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here); -} - -foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE, - svm_fifo_enqueue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma); - +#ifndef CLIB_MARCH_VARIANT int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here); -#else - static int (*fp) (svm_fifo_t *, u32, const u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_from_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes, + copy_from_here); } +#endif /** * Enqueue a future segment. @@ -557,11 +535,8 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, * Returns 0 of the entire segment was copied * Returns -1 if none of the segment was copied due to lack of space */ -static int -svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, - u32 offset, - u32 required_bytes, - u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, + u32 offset, u32 required_bytes, u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, normalized_offset; @@ -609,14 +584,15 @@ svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, return (0); } +#ifndef CLIB_MARCH_VARIANT int -svm_fifo_enqueue_with_offset (svm_fifo_t * f, - u32 offset, - u32 required_bytes, u8 * copy_from_here) +svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes, + u8 * copy_from_here) { - return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes, - copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset, + required_bytes, + copy_from_here); } void @@ -633,9 +609,10 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk); } } +#endif -static int -svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, + u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -687,43 +664,18 @@ svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) return (total_copy_bytes); } -static int -svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) -{ - return svm_fifo_dequeue_internal (f, max_bytes, copy_here); -} - -#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE, - svm_fifo_dequeue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes, + copy_here); } +#endif -static int -svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, + u32 max_bytes, u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, real_head; @@ -760,32 +712,14 @@ svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, return total_copy_bytes; } -#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, relative_offset, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_peek_ma_multiarch_select (); - - return (*fp) (f, relative_offset, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes, + copy_here); } int @@ -835,6 +769,53 @@ svm_fifo_dequeue_drop_all (svm_fifo_t * f) __sync_fetch_and_sub (&f->cursize, f->cursize); } +int +svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs) +{ + u32 cursize, nitems; + + /* read cursize, which can only increase while we're working */ + cursize = svm_fifo_max_dequeue (f); + if (PREDICT_FALSE (cursize == 0)) + return -2; + + nitems = f->nitems; + + fs[0].len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize; + fs[0].data = f->data + f->head; + + if (fs[0].len < cursize) + { + fs[1].len = cursize - fs[0].len; + fs[1].data = f->data; + } + else + { + fs[1].len = 0; + fs[1].data = 0; + } + return cursize; +} + +void +svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs) +{ + u32 total_drop_bytes; + + ASSERT (fs[0].data == f->data + f->head); + if (fs[1].len) + { + f->head = fs[1].len; + total_drop_bytes = fs[0].len + fs[1].len; + } + else + { + f->head = (f->head + fs[0].len) % f->nitems; + total_drop_bytes = fs[0].len; + } + __sync_fetch_and_sub (&f->cursize, total_drop_bytes); +} + u32 svm_fifo_number_ooo_segments (svm_fifo_t * f) { @@ -856,6 +837,7 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer) f->head = f->tail = pointer % f->nitems; } +#endif /* * fd.io coding-style-patch-verification: ON *