X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=4eae0a1c7e68b38c6a64ca267e157841e39003c9;hb=326b81e30e63a8296df51d85e6514356cd737225;hp=fc2189c51764a306c02f9665990e5860d2e38912;hpb=3eb5062b40feb3002de09a3caff86232d6e1adea;p=vpp.git diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index fc2189c5176..4eae0a1c7e6 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -50,13 +50,17 @@ ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s) return (s->start + s->length) % f->nitems; } +#ifndef CLIB_MARCH_VARIANT + u8 * format_ooo_segment (u8 * s, va_list * args) { + svm_fifo_t *f = va_arg (*args, svm_fifo_t *); ooo_segment_t *seg = va_arg (*args, ooo_segment_t *); - - s = format (s, "pos %u, len %u, next %d, prev %d", - seg->start, seg->length, seg->next, seg->prev); + u32 normalized_start = (seg->start + f->nitems - f->tail) % f->nitems; + s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start, + (normalized_start + seg->length) % f->nitems, seg->length, + seg->next, seg->prev); return s; } @@ -154,7 +158,7 @@ format_ooo_list (u8 * s, va_list * args) while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX) { seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index); - s = format (s, " %U\n", format_ooo_segment, seg); + s = format (s, " %U\n", format_ooo_segment, f, seg); ooo_segment_index = seg->next; } @@ -167,13 +171,17 @@ format_svm_fifo (u8 * s, va_list * args) svm_fifo_t *f = va_arg (*args, svm_fifo_t *); int verbose = va_arg (*args, int); + if (!s) + return s; + s = format (s, "cursize %u nitems %u has_event %d\n", f->cursize, f->nitems, f->has_event); - s = format (s, " head %d tail %d\n", f->head, f->tail); + s = format (s, " head %d tail %d segment manager %u\n", f->head, f->tail, + f->segment_manager); if (verbose > 1) s = format - (s, " server session %d thread %d client session %d thread %d\n", + (s, " vpp session %d thread %d app session %d thread %d\n", f->master_session_index, f->master_thread_index, f->client_session_index, f->client_thread_index); @@ -192,8 +200,11 @@ svm_fifo_t * svm_fifo_create (u32 data_size_in_bytes) { svm_fifo_t *f; + u32 rounded_data_size; - f = clib_mem_alloc_aligned_or_null (sizeof (*f) + data_size_in_bytes, + /* always round fifo data size to the next highest power-of-two */ + rounded_data_size = (1 << (max_log2 (data_size_in_bytes))); + f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size, CLIB_CACHE_LINE_BYTES); if (f == 0) return 0; @@ -201,15 +212,23 @@ svm_fifo_create (u32 data_size_in_bytes) memset (f, 0, sizeof (*f)); f->nitems = data_size_in_bytes; f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX; + f->refcnt = 1; return (f); } void svm_fifo_free (svm_fifo_t * f) { - pool_free (f->ooo_segments); - clib_mem_free (f); + ASSERT (f->refcnt > 0); + + if (--f->refcnt == 0) + { + pool_free (f->ooo_segments); + clib_mem_free (f); + } } +#endif always_inline ooo_segment_t * ooo_segment_new (svm_fifo_t * f, u32 start, u32 length) @@ -432,12 +451,12 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued) } } - ASSERT (bytes >= 0 && bytes <= f->nitems); + ASSERT (bytes <= f->nitems); return bytes; } -static int -svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -447,7 +466,7 @@ svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) f->ooos_newest = OOO_SEGMENT_INVALID_INDEX; if (PREDICT_FALSE (cursize == f->nitems)) - return -2; /* fifo stuffed */ + return SVM_FIFO_FULL; nitems = f->nitems; @@ -499,39 +518,15 @@ svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) return (total_copy_bytes); } -#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \ - { return fn (f, max_bytes, copy_from_here);} - -static int -svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes, - u8 * copy_from_here) -{ - return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here); -} - -foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE, - svm_fifo_enqueue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma); - +#ifndef CLIB_MARCH_VARIANT int -svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) +svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here); -#else - static int (*fp) (svm_fifo_t *, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_from_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes, + copy_from_here); } +#endif /** * Enqueue a future segment. @@ -540,15 +535,11 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) * Returns 0 of the entire segment was copied * Returns -1 if none of the segment was copied due to lack of space */ -static int -svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, - u32 offset, - u32 required_bytes, - u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, + u32 offset, u32 required_bytes, u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, normalized_offset; - u32 offset_from_tail; f->ooos_newest = OOO_SEGMENT_INVALID_INDEX; @@ -561,8 +552,7 @@ svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, normalized_offset = (f->tail + offset) % nitems; /* Will this request fit? */ - offset_from_tail = (nitems + normalized_offset - f->tail) % nitems; - if ((required_bytes + offset_from_tail) > (nitems - cursize)) + if ((required_bytes + offset) > (nitems - cursize)) return -1; svm_fifo_trace_add (f, offset, required_bytes, 1); @@ -594,19 +584,35 @@ svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f, return (0); } +#ifndef CLIB_MARCH_VARIANT int -svm_fifo_enqueue_with_offset (svm_fifo_t * f, - u32 offset, - u32 required_bytes, u8 * copy_from_here) +svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes, + u8 * copy_from_here) { - return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes, - copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset, + required_bytes, + copy_from_here); } +void +svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) +{ + u32 first_chunk; + first_chunk = f->nitems - f->head; + ASSERT (len <= f->nitems); + if (len <= first_chunk) + clib_memcpy (&f->data[f->head], data, len); + else + { + clib_memcpy (&f->data[f->head], data, first_chunk); + clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk); + } +} +#endif -static int -svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, + u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -658,43 +664,18 @@ svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) return (total_copy_bytes); } -static int -svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) -{ - return svm_fifo_dequeue_internal (f, max_bytes, copy_here); -} - -#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE, - svm_fifo_dequeue_nowait_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select (); - - return (*fp) (f, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes, + copy_here); } +#endif -static int -svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, + u32 max_bytes, u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, real_head; @@ -731,32 +712,14 @@ svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, return total_copy_bytes; } -#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \ - u8 * copy_here) \ - { return fn (f, relative_offset, max_bytes, copy_here);} - -foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma); -CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma); +#ifndef CLIB_MARCH_VARIANT int svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, u8 * copy_here) { -#if CLIB_DEBUG > 0 - return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here); -#else - static int (*fp) (svm_fifo_t *, u32, u32, u8 *); - - if (PREDICT_FALSE (fp == 0)) - fp = (void *) svm_fifo_peek_ma_multiarch_select (); - - return (*fp) (f, relative_offset, max_bytes, copy_here); -#endif + return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes, + copy_here); } int @@ -799,6 +762,60 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes) return total_drop_bytes; } +void +svm_fifo_dequeue_drop_all (svm_fifo_t * f) +{ + f->head = f->tail; + __sync_fetch_and_sub (&f->cursize, f->cursize); +} + +int +svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs) +{ + u32 cursize, nitems; + + /* read cursize, which can only increase while we're working */ + cursize = svm_fifo_max_dequeue (f); + if (PREDICT_FALSE (cursize == 0)) + return -2; + + nitems = f->nitems; + + fs[0].len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize; + fs[0].data = f->data + f->head; + + if (fs[0].len < cursize) + { + fs[1].len = cursize - fs[0].len; + fs[1].data = f->data; + } + else + { + fs[1].len = 0; + fs[1].data = 0; + } + return cursize; +} + +void +svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs) +{ + u32 total_drop_bytes; + + ASSERT (fs[0].data == f->data + f->head); + if (fs[1].len) + { + f->head = fs[1].len; + total_drop_bytes = fs[0].len + fs[1].len; + } + else + { + f->head = (f->head + fs[0].len) % f->nitems; + total_drop_bytes = fs[0].len; + } + __sync_fetch_and_sub (&f->cursize, total_drop_bytes); +} + u32 svm_fifo_number_ooo_segments (svm_fifo_t * f) { @@ -820,6 +837,7 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer) f->head = f->tail = pointer % f->nitems; } +#endif /* * fd.io coding-style-patch-verification: ON *