X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=e7a1188d59157956ff7840dc1f15b56fcbd59484;hb=fa76a76;hp=6a9c0df676f7e23bb6f40d00393ce0effad56fc2;hpb=3aa7af3bd703325619634ef9938cf990be3e6f24;p=vpp.git diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 6a9c0df676f..e7a1188d591 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -16,41 +16,6 @@ #include #include -#if __x86_64__ && CLIB_DEBUG == 0 -#define foreach_march_variant_runtime(macro, _args...) \ - macro(avx512, avx512f, _args) \ - macro(avx2, avx2, _args) -#else -#define foreach_march_variant_runtime(macro, _args...) -#endif - -#define CLIB_MARCH_ARCH_CHECK(arch, archname, fn) \ - if (clib_cpu_supports_ ## archname ()) \ - return & fn ## _ ##arch; - -#define CLIB_MARCH_SELECT_FN(fn,...) \ - __VA_ARGS__ void * fn ## _multiarch_select(void) \ -{ \ - foreach_march_variant_runtime(CLIB_MARCH_ARCH_CHECK, fn) \ - return & fn; \ -} - -#define DEFINE_WEAK_FN(_arch, _fn, _args...) \ - int __attribute__((weak)) _fn##_ma_##_arch(_args); -#define DEFINE_FN_HELPER(arch, archname, macro, _args...) \ - macro(arch, _args) - -#ifndef CLIB_MARCH_VARIANT -#define MARCH_FN(fn, _args...) \ - static void * (*fn ## _selected) (void); \ - foreach_march_variant_runtime (DEFINE_FN_HELPER, DEFINE_WEAK_FN, fn, \ - _args); \ - static inline int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) -#else -#define MARCH_FN(fn, _args...) \ - int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args) -#endif - static inline u8 position_lt (svm_fifo_t * f, u32 a, u32 b) { @@ -144,7 +109,7 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose) #endif dummy_fifo = svm_fifo_create (f->nitems); - memset (f->data, 0xFF, f->nitems); + clib_memset (f->data, 0xFF, f->nitems); vec_validate (data, f->nitems); for (i = 0; i < vec_len (data); i++) @@ -187,13 +152,15 @@ u8 * format_ooo_list (u8 * s, va_list * args) { svm_fifo_t *f = va_arg (*args, svm_fifo_t *); + u32 indent = va_arg (*args, u32); u32 ooo_segment_index = f->ooos_list_head; ooo_segment_t *seg; while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX) { seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index); - s = format (s, " %U\n", format_ooo_segment, f, seg); + s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment, + f, seg); ooo_segment_index = seg->next; } @@ -205,27 +172,30 @@ format_svm_fifo (u8 * s, va_list * args) { svm_fifo_t *f = va_arg (*args, svm_fifo_t *); int verbose = va_arg (*args, int); + u32 indent; if (!s) return s; + indent = format_get_indent (s); s = format (s, "cursize %u nitems %u has_event %d\n", f->cursize, f->nitems, f->has_event); - s = format (s, " head %d tail %d segment manager %u\n", f->head, f->tail, - f->segment_manager); + s = format (s, "%Uhead %d tail %d segment manager %u\n", format_white_space, + indent, f->head, f->tail, f->segment_manager); if (verbose > 1) - s = format - (s, " vpp session %d thread %d app session %d thread %d\n", - f->master_session_index, f->master_thread_index, - f->client_session_index, f->client_thread_index); + s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n", + format_white_space, indent, f->master_session_index, + f->master_thread_index, f->client_session_index, + f->client_thread_index); if (verbose) { - s = format (s, " ooo pool %d active elts newest %u\n", - pool_elts (f->ooo_segments), f->ooos_newest); + s = format (s, "%Uooo pool %d active elts newest %u\n", + format_white_space, indent, pool_elts (f->ooo_segments), + f->ooos_newest); if (svm_fifo_has_ooo_data (f)) - s = format (s, " %U", format_ooo_list, f, verbose); + s = format (s, " %U", format_ooo_list, f, indent, verbose); } return s; } @@ -244,9 +214,11 @@ svm_fifo_create (u32 data_size_in_bytes) if (f == 0) return 0; - memset (f, 0, sizeof (*f)); + clib_memset (f, 0, sizeof (*f)); f->nitems = data_size_in_bytes; f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX; + f->segment_index = SVM_FIFO_INVALID_INDEX; f->refcnt = 1; return (f); } @@ -489,8 +461,8 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued) return bytes; } -MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes, - const u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, + const u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -514,7 +486,7 @@ MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes, first_copy_bytes = ((nitems - f->tail) < total_copy_bytes) ? (nitems - f->tail) : total_copy_bytes; - clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes); + clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes); f->tail += first_copy_bytes; f->tail = (f->tail == nitems) ? 0 : f->tail; @@ -522,8 +494,9 @@ MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes, second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes, - second_copy_bytes); + clib_memcpy_fast (&f->data[f->tail], + copy_from_here + first_copy_bytes, + second_copy_bytes); f->tail += second_copy_bytes; f->tail = (f->tail == nitems) ? 0 : f->tail; } @@ -547,7 +520,7 @@ MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes, /* Atomically increase the queue length */ ASSERT (cursize + total_copy_bytes <= nitems); - __sync_fetch_and_add (&f->cursize, total_copy_bytes); + clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes); return (total_copy_bytes); } @@ -557,8 +530,8 @@ int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here) { - return ((int (*)(svm_fifo_t * f, u32, const u8 *)) - (*svm_fifo_enqueue_nowait_selected)) (f, max_bytes, copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes, + copy_from_here); } #endif @@ -569,8 +542,8 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, * Returns 0 of the entire segment was copied * Returns -1 if none of the segment was copied due to lack of space */ -MARCH_FN (svm_fifo_enqueue_with_offset, svm_fifo_t * f, - u32 offset, u32 required_bytes, u8 * copy_from_here) +CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, + u32 offset, u32 required_bytes, u8 * copy_from_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, normalized_offset; @@ -600,7 +573,8 @@ MARCH_FN (svm_fifo_enqueue_with_offset, svm_fifo_t * f, first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes) ? (nitems - normalized_offset) : total_copy_bytes; - clib_memcpy (&f->data[normalized_offset], copy_from_here, first_copy_bytes); + clib_memcpy_fast (&f->data[normalized_offset], copy_from_here, + first_copy_bytes); /* Number of bytes in second copy segment, if any */ second_copy_bytes = total_copy_bytes - first_copy_bytes; @@ -611,8 +585,8 @@ MARCH_FN (svm_fifo_enqueue_with_offset, svm_fifo_t * f, ASSERT (normalized_offset == 0); - clib_memcpy (&f->data[normalized_offset], - copy_from_here + first_copy_bytes, second_copy_bytes); + clib_memcpy_fast (&f->data[normalized_offset], + copy_from_here + first_copy_bytes, second_copy_bytes); } return (0); @@ -624,10 +598,9 @@ int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes, u8 * copy_from_here) { - return ((int (*)(svm_fifo_t * f, u32, u32, u8 *)) - (*svm_fifo_enqueue_with_offset_selected)) (f, offset, - required_bytes, - copy_from_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset, + required_bytes, + copy_from_here); } void @@ -637,17 +610,17 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) first_chunk = f->nitems - f->head; ASSERT (len <= f->nitems); if (len <= first_chunk) - clib_memcpy (&f->data[f->head], data, len); + clib_memcpy_fast (&f->data[f->head], data, len); else { - clib_memcpy (&f->data[f->head], data, first_chunk); - clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk); + clib_memcpy_fast (&f->data[f->head], data, first_chunk); + clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk); } } #endif -MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, + u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems; @@ -667,7 +640,7 @@ MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, /* Number of bytes in first copy segment */ first_copy_bytes = ((nitems - f->head) < total_copy_bytes) ? (nitems - f->head) : total_copy_bytes; - clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes); + clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes); f->head += first_copy_bytes; f->head = (f->head == nitems) ? 0 : f->head; @@ -675,8 +648,8 @@ MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - clib_memcpy (copy_here + first_copy_bytes, - &f->data[f->head], second_copy_bytes); + clib_memcpy_fast (copy_here + first_copy_bytes, + &f->data[f->head], second_copy_bytes); f->head += second_copy_bytes; f->head = (f->head == nitems) ? 0 : f->head; } @@ -694,7 +667,7 @@ MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, ASSERT (f->head <= nitems); ASSERT (cursize >= total_copy_bytes); - __sync_fetch_and_sub (&f->cursize, total_copy_bytes); + clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes); return (total_copy_bytes); } @@ -704,13 +677,13 @@ MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes, int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here) { - return ((int (*)(svm_fifo_t * f, u32, u8 *)) - (*svm_fifo_dequeue_nowait_selected)) (f, max_bytes, copy_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes, + copy_here); } #endif -MARCH_FN (svm_fifo_peek, svm_fifo_t * f, u32 relative_offset, u32 max_bytes, - u8 * copy_here) +CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, + u32 max_bytes, u8 * copy_here) { u32 total_copy_bytes, first_copy_bytes, second_copy_bytes; u32 cursize, nitems, real_head; @@ -734,14 +707,14 @@ MARCH_FN (svm_fifo_peek, svm_fifo_t * f, u32 relative_offset, u32 max_bytes, first_copy_bytes = ((nitems - real_head) < total_copy_bytes) ? (nitems - real_head) : total_copy_bytes; - clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes); + clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes); /* Number of bytes in second copy segment, if any */ second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - clib_memcpy (copy_here + first_copy_bytes, &f->data[0], - second_copy_bytes); + clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0], + second_copy_bytes); } } return total_copy_bytes; @@ -753,9 +726,8 @@ int svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes, u8 * copy_here) { - return ((int (*)(svm_fifo_t * f, u32, u32, u8 *)) - (*svm_fifo_peek_selected)) (f, relative_offset, max_bytes, - copy_here); + return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes, + copy_here); } int @@ -793,7 +765,7 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes) ASSERT (f->head <= nitems); ASSERT (cursize >= total_drop_bytes); - __sync_fetch_and_sub (&f->cursize, total_drop_bytes); + clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes); return total_drop_bytes; } @@ -802,7 +774,7 @@ void svm_fifo_dequeue_drop_all (svm_fifo_t * f) { f->head = f->tail; - __sync_fetch_and_sub (&f->cursize, f->cursize); + clib_atomic_fetch_sub_rel (&f->cursize, f->cursize); } int @@ -849,7 +821,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs) f->head = (f->head + fs[0].len) % f->nitems; total_drop_bytes = fs[0].len; } - __sync_fetch_and_sub (&f->cursize, total_drop_bytes); + clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes); } u32 @@ -873,23 +845,6 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer) f->head = f->tail = pointer % f->nitems; } -#define foreach_svm_fifo_march_fn \ - _(svm_fifo_enqueue_nowait) \ - _(svm_fifo_enqueue_with_offset) \ - _(svm_fifo_dequeue_nowait) \ - _(svm_fifo_peek) \ - -#define _(_fn, _args...) CLIB_MARCH_SELECT_FN(_fn ## _ma); -foreach_svm_fifo_march_fn -#undef _ -void __clib_constructor -svm_fifo_select_march_fns (void) -{ -#define _(_fn, _args...) _fn ## _selected = _fn ## _ma_multiarch_select (); - foreach_svm_fifo_march_fn -#undef _ -} - #endif /* * fd.io coding-style-patch-verification: ON