#include <svm/svm_fifo.h>
#include <vppinfra/cpu.h>
-#if __x86_64__ && CLIB_DEBUG == 0
-#define foreach_march_variant_runtime(macro, _args...) \
- macro(avx512, avx512f, _args) \
- macro(avx2, avx2, _args)
-#else
-#define foreach_march_variant_runtime(macro, _args...)
-#endif
-
-#define CLIB_MARCH_ARCH_CHECK(arch, archname, fn) \
- if (clib_cpu_supports_ ## archname ()) \
- return & fn ## _ ##arch;
-
-#define CLIB_MARCH_SELECT_FN(fn,...) \
- __VA_ARGS__ void * fn ## _multiarch_select(void) \
-{ \
- foreach_march_variant_runtime(CLIB_MARCH_ARCH_CHECK, fn) \
- return & fn; \
-}
-
-#define DEFINE_WEAK_FN(_arch, _fn, _args...) \
- int __attribute__((weak)) _fn##_ma_##_arch(_args);
-#define DEFINE_FN_HELPER(arch, archname, macro, _args...) \
- macro(arch, _args)
-
-#ifndef CLIB_MARCH_VARIANT
-#define MARCH_FN(fn, _args...) \
- static void * (*fn ## _selected) (void); \
- foreach_march_variant_runtime (DEFINE_FN_HELPER, DEFINE_WEAK_FN, fn, \
- _args); \
- static inline int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
-#else
-#define MARCH_FN(fn, _args...) \
- int CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
-#endif
-
static inline u8
position_lt (svm_fifo_t * f, u32 a, u32 b)
{
#endif
dummy_fifo = svm_fifo_create (f->nitems);
- memset (f->data, 0xFF, f->nitems);
+ clib_memset (f->data, 0xFF, f->nitems);
vec_validate (data, f->nitems);
for (i = 0; i < vec_len (data); i++)
format_ooo_list (u8 * s, va_list * args)
{
svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
+ u32 indent = va_arg (*args, u32);
u32 ooo_segment_index = f->ooos_list_head;
ooo_segment_t *seg;
while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
{
seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
- s = format (s, " %U\n", format_ooo_segment, f, seg);
+ s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
+ f, seg);
ooo_segment_index = seg->next;
}
{
svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
int verbose = va_arg (*args, int);
+ u32 indent;
if (!s)
return s;
+ indent = format_get_indent (s);
s = format (s, "cursize %u nitems %u has_event %d\n",
f->cursize, f->nitems, f->has_event);
- s = format (s, " head %d tail %d segment manager %u\n", f->head, f->tail,
- f->segment_manager);
+ s = format (s, "%Uhead %d tail %d segment manager %u\n", format_white_space,
+ indent, f->head, f->tail, f->segment_manager);
if (verbose > 1)
- s = format
- (s, " vpp session %d thread %d app session %d thread %d\n",
- f->master_session_index, f->master_thread_index,
- f->client_session_index, f->client_thread_index);
+ s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
+ format_white_space, indent, f->master_session_index,
+ f->master_thread_index, f->client_session_index,
+ f->client_thread_index);
if (verbose)
{
- s = format (s, " ooo pool %d active elts newest %u\n",
- pool_elts (f->ooo_segments), f->ooos_newest);
+ s = format (s, "%Uooo pool %d active elts newest %u\n",
+ format_white_space, indent, pool_elts (f->ooo_segments),
+ f->ooos_newest);
if (svm_fifo_has_ooo_data (f))
- s = format (s, " %U", format_ooo_list, f, verbose);
+ s = format (s, " %U", format_ooo_list, f, indent, verbose);
}
return s;
}
if (f == 0)
return 0;
- memset (f, 0, sizeof (*f));
+ clib_memset (f, 0, sizeof (*f));
f->nitems = data_size_in_bytes;
f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
f->refcnt = 1;
return (f);
}
return bytes;
}
-MARCH_FN (svm_fifo_enqueue_nowait, svm_fifo_t * f, u32 max_bytes,
- const u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
+ const u8 * copy_from_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems;
first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
? (nitems - f->tail) : total_copy_bytes;
- clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes);
+ clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes);
f->tail += first_copy_bytes;
f->tail = (f->tail == nitems) ? 0 : f->tail;
second_copy_bytes = total_copy_bytes - first_copy_bytes;
if (second_copy_bytes)
{
- clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes,
- second_copy_bytes);
+ clib_memcpy_fast (&f->data[f->tail],
+ copy_from_here + first_copy_bytes,
+ second_copy_bytes);
f->tail += second_copy_bytes;
f->tail = (f->tail == nitems) ? 0 : f->tail;
}
/* Atomically increase the queue length */
ASSERT (cursize + total_copy_bytes <= nitems);
- __sync_fetch_and_add (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
const u8 * copy_from_here)
{
- return ((int (*)(svm_fifo_t * f, u32, const u8 *))
- (*svm_fifo_enqueue_nowait_selected)) (f, max_bytes, copy_from_here);
+ return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes,
+ copy_from_here);
}
#endif
* Returns 0 of the entire segment was copied
* Returns -1 if none of the segment was copied due to lack of space
*/
-MARCH_FN (svm_fifo_enqueue_with_offset, svm_fifo_t * f,
- u32 offset, u32 required_bytes, u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
+ u32 offset, u32 required_bytes, u8 * copy_from_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems, normalized_offset;
first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
? (nitems - normalized_offset) : total_copy_bytes;
- clib_memcpy (&f->data[normalized_offset], copy_from_here, first_copy_bytes);
+ clib_memcpy_fast (&f->data[normalized_offset], copy_from_here,
+ first_copy_bytes);
/* Number of bytes in second copy segment, if any */
second_copy_bytes = total_copy_bytes - first_copy_bytes;
ASSERT (normalized_offset == 0);
- clib_memcpy (&f->data[normalized_offset],
- copy_from_here + first_copy_bytes, second_copy_bytes);
+ clib_memcpy_fast (&f->data[normalized_offset],
+ copy_from_here + first_copy_bytes, second_copy_bytes);
}
return (0);
svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
u8 * copy_from_here)
{
- return ((int (*)(svm_fifo_t * f, u32, u32, u8 *))
- (*svm_fifo_enqueue_with_offset_selected)) (f, offset,
- required_bytes,
- copy_from_here);
+ return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset,
+ required_bytes,
+ copy_from_here);
}
void
first_chunk = f->nitems - f->head;
ASSERT (len <= f->nitems);
if (len <= first_chunk)
- clib_memcpy (&f->data[f->head], data, len);
+ clib_memcpy_fast (&f->data[f->head], data, len);
else
{
- clib_memcpy (&f->data[f->head], data, first_chunk);
- clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk);
+ clib_memcpy_fast (&f->data[f->head], data, first_chunk);
+ clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
}
}
#endif
-MARCH_FN (svm_fifo_dequeue_nowait, svm_fifo_t * f, u32 max_bytes,
- u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
+ u8 * copy_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems;
/* Number of bytes in first copy segment */
first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
? (nitems - f->head) : total_copy_bytes;
- clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes);
+ clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes);
f->head += first_copy_bytes;
f->head = (f->head == nitems) ? 0 : f->head;
second_copy_bytes = total_copy_bytes - first_copy_bytes;
if (second_copy_bytes)
{
- clib_memcpy (copy_here + first_copy_bytes,
- &f->data[f->head], second_copy_bytes);
+ clib_memcpy_fast (copy_here + first_copy_bytes,
+ &f->data[f->head], second_copy_bytes);
f->head += second_copy_bytes;
f->head = (f->head == nitems) ? 0 : f->head;
}
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_copy_bytes);
- __sync_fetch_and_sub (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
int
svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
{
- return ((int (*)(svm_fifo_t * f, u32, u8 *))
- (*svm_fifo_dequeue_nowait_selected)) (f, max_bytes, copy_here);
+ return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes,
+ copy_here);
}
#endif
-MARCH_FN (svm_fifo_peek, svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
- u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
+ u32 max_bytes, u8 * copy_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems, real_head;
first_copy_bytes =
((nitems - real_head) < total_copy_bytes) ?
(nitems - real_head) : total_copy_bytes;
- clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes);
+ clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes);
/* Number of bytes in second copy segment, if any */
second_copy_bytes = total_copy_bytes - first_copy_bytes;
if (second_copy_bytes)
{
- clib_memcpy (copy_here + first_copy_bytes, &f->data[0],
- second_copy_bytes);
+ clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
+ second_copy_bytes);
}
}
return total_copy_bytes;
svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
u8 * copy_here)
{
- return ((int (*)(svm_fifo_t * f, u32, u32, u8 *))
- (*svm_fifo_peek_selected)) (f, relative_offset, max_bytes,
- copy_here);
+ return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes,
+ copy_here);
}
int
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_drop_bytes);
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
return total_drop_bytes;
}
svm_fifo_dequeue_drop_all (svm_fifo_t * f)
{
f->head = f->tail;
- __sync_fetch_and_sub (&f->cursize, f->cursize);
+ clib_atomic_fetch_sub_rel (&f->cursize, f->cursize);
}
int
f->head = (f->head + fs[0].len) % f->nitems;
total_drop_bytes = fs[0].len;
}
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
}
u32
f->head = f->tail = pointer % f->nitems;
}
-#define foreach_svm_fifo_march_fn \
- _(svm_fifo_enqueue_nowait) \
- _(svm_fifo_enqueue_with_offset) \
- _(svm_fifo_dequeue_nowait) \
- _(svm_fifo_peek) \
-
-#define _(_fn, _args...) CLIB_MARCH_SELECT_FN(_fn ## _ma);
-foreach_svm_fifo_march_fn
-#undef _
-void __clib_constructor
-svm_fifo_select_march_fns (void)
-{
-#define _(_fn, _args...) _fn ## _selected = _fn ## _ma_multiarch_select ();
- foreach_svm_fifo_march_fn
-#undef _
-}
-
#endif
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff