return (s->start + s->length) % f->nitems;
}
+#ifndef CLIB_MARCH_VARIANT
+
u8 *
format_ooo_segment (u8 * s, va_list * args)
{
memset (f, 0, sizeof (*f));
f->nitems = data_size_in_bytes;
f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
f->refcnt = 1;
return (f);
}
clib_mem_free (f);
}
}
+#endif
always_inline ooo_segment_t *
ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
return bytes;
}
-static int
-svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes,
- const u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
+ const u8 * copy_from_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems;
return (total_copy_bytes);
}
-#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \
- uword \
- __attribute__ ((flatten)) \
- __attribute__ ((target (tgt))) \
- CLIB_CPU_OPTIMIZED \
- fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \
- { return fn (f, max_bytes, copy_from_here);}
-
-static int
-svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes,
- const u8 * copy_from_here)
-{
- return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here);
-}
-
-foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE,
- svm_fifo_enqueue_nowait_ma);
-CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma);
-
+#ifndef CLIB_MARCH_VARIANT
int
svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
const u8 * copy_from_here)
{
-#if CLIB_DEBUG > 0
- return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here);
-#else
- static int (*fp) (svm_fifo_t *, u32, const u8 *);
-
- if (PREDICT_FALSE (fp == 0))
- fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select ();
-
- return (*fp) (f, max_bytes, copy_from_here);
-#endif
+ return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes,
+ copy_from_here);
}
+#endif
/**
* Enqueue a future segment.
* Returns 0 of the entire segment was copied
* Returns -1 if none of the segment was copied due to lack of space
*/
-static int
-svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f,
- u32 offset,
- u32 required_bytes,
- u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
+ u32 offset, u32 required_bytes, u8 * copy_from_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems, normalized_offset;
return (0);
}
+#ifndef CLIB_MARCH_VARIANT
int
-svm_fifo_enqueue_with_offset (svm_fifo_t * f,
- u32 offset,
- u32 required_bytes, u8 * copy_from_here)
+svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
+ u8 * copy_from_here)
{
- return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes,
- copy_from_here);
+ return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset,
+ required_bytes,
+ copy_from_here);
}
void
clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk);
}
}
+#endif
-static int
-svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
+ u8 * copy_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems;
return (total_copy_bytes);
}
-static int
-svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
-{
- return svm_fifo_dequeue_internal (f, max_bytes, copy_here);
-}
-
-#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \
- uword \
- __attribute__ ((flatten)) \
- __attribute__ ((target (tgt))) \
- CLIB_CPU_OPTIMIZED \
- fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, \
- u8 * copy_here) \
- { return fn (f, max_bytes, copy_here);}
-
-foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE,
- svm_fifo_dequeue_nowait_ma);
-CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma);
+#ifndef CLIB_MARCH_VARIANT
int
svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
{
-#if CLIB_DEBUG > 0
- return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here);
-#else
- static int (*fp) (svm_fifo_t *, u32, u8 *);
-
- if (PREDICT_FALSE (fp == 0))
- fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select ();
-
- return (*fp) (f, max_bytes, copy_here);
-#endif
+ return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes,
+ copy_here);
}
+#endif
-static int
-svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
- u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
+ u32 max_bytes, u8 * copy_here)
{
u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
u32 cursize, nitems, real_head;
return total_copy_bytes;
}
-#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt) \
- uword \
- __attribute__ ((flatten)) \
- __attribute__ ((target (tgt))) \
- CLIB_CPU_OPTIMIZED \
- fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \
- u8 * copy_here) \
- { return fn (f, relative_offset, max_bytes, copy_here);}
-
-foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma);
-CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma);
+#ifndef CLIB_MARCH_VARIANT
int
svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
u8 * copy_here)
{
-#if CLIB_DEBUG > 0
- return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here);
-#else
- static int (*fp) (svm_fifo_t *, u32, u32, u8 *);
-
- if (PREDICT_FALSE (fp == 0))
- fp = (void *) svm_fifo_peek_ma_multiarch_select ();
-
- return (*fp) (f, relative_offset, max_bytes, copy_here);
-#endif
+ return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes,
+ copy_here);
}
int
__sync_fetch_and_sub (&f->cursize, f->cursize);
}
+int
+svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs)
+{
+ u32 cursize, nitems;
+
+ /* read cursize, which can only increase while we're working */
+ cursize = svm_fifo_max_dequeue (f);
+ if (PREDICT_FALSE (cursize == 0))
+ return -2;
+
+ nitems = f->nitems;
+
+ fs[0].len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize;
+ fs[0].data = f->data + f->head;
+
+ if (fs[0].len < cursize)
+ {
+ fs[1].len = cursize - fs[0].len;
+ fs[1].data = f->data;
+ }
+ else
+ {
+ fs[1].len = 0;
+ fs[1].data = 0;
+ }
+ return cursize;
+}
+
+void
+svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
+{
+ u32 total_drop_bytes;
+
+ ASSERT (fs[0].data == f->data + f->head);
+ if (fs[1].len)
+ {
+ f->head = fs[1].len;
+ total_drop_bytes = fs[0].len + fs[1].len;
+ }
+ else
+ {
+ f->head = (f->head + fs[0].len) % f->nitems;
+ total_drop_bytes = fs[0].len;
+ }
+ __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+}
+
u32
svm_fifo_number_ooo_segments (svm_fifo_t * f)
{
f->head = f->tail = pointer % f->nitems;
}
+#endif
/*
* fd.io coding-style-patch-verification: ON
*
Code Review / vpp.git / blobdiff