#define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
do { \
+ ASSERT (bi0 != 0); \
+ ASSERT (bi1 != 0); \
int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \
\
if (PREDICT_FALSE (enqueue_code != 0)) \
#define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
do { \
+ ASSERT (bi0 != 0); \
+ ASSERT (bi1 != 0); \
+ ASSERT (bi2 != 0); \
+ ASSERT (bi3 != 0); \
/* After the fact: check the [speculative] enqueue to "next" */ \
u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1) \
| (next_index ^ next2) | (next_index ^ next3); \
*/
#define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
do { \
+ ASSERT (bi0 != 0); \
if (PREDICT_FALSE (next0 != next_index)) \
{ \
vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
vlib_prefetch_buffer_header (p2, LOAD);
vlib_prefetch_buffer_header (p3, LOAD);
- CLIB_PREFETCH (p2->data, 64, LOAD);
- CLIB_PREFETCH (p3->data, 64, LOAD);
+ clib_prefetch_load (p2->data);
+ clib_prefetch_load (p3->data);
}
pi0 = to_next[0] = from[0];
return frame->n_vectors;
}
+/* Minimum size for the 'buffers' and 'nexts' arrays to be used when calling
+ * vlib_buffer_enqueue_to_next().
+ * Because of optimizations, vlib_buffer_enqueue_to_next() will access
+ * past 'count' elements in the 'buffers' and 'nexts' arrays, IOW it
+ * will overflow.
+ * Those overflow elements are ignored in the final result so they do not
+ * need to be properly initialized, however if the array is allocated right
+ * before the end of a page and the next page is not mapped, accessing the
+ * overflow elements will trigger a segfault. */
+#define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n) round_pow2 ((n), 64)
+
+static_always_inline void
+vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
+ u32 * buffers, u16 * nexts, uword count)
+{
+ vlib_buffer_enqueue_to_next_fn_t *fn;
+ fn = vlib_buffer_func_main.buffer_enqueue_to_next_fn;
+ (fn) (vm, node, buffers, nexts, count);
+}
+
+static_always_inline void
+vlib_buffer_enqueue_to_next_vec (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 **buffers, u16 **nexts, uword count)
+{
+ const u32 bl = vec_len (*buffers), nl = vec_len (*nexts);
+ const u32 c = VLIB_BUFFER_ENQUEUE_MIN_SIZE (count);
+ ASSERT (bl >= count && nl >= count);
+ vec_validate (*buffers, c);
+ vec_validate (*nexts, c);
+ vlib_buffer_enqueue_to_next (vm, node, *buffers, *nexts, count);
+ vec_set_len (*buffers, bl);
+ vec_set_len (*nexts, nl);
+}
+
+static_always_inline void
+vlib_buffer_enqueue_to_single_next (vlib_main_t * vm,
+ vlib_node_runtime_t * node, u32 * buffers,
+ u16 next_index, u32 count)
+{
+ vlib_buffer_enqueue_to_single_next_fn_t *fn;
+ fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_fn;
+ (fn) (vm, node, buffers, next_index, count);
+}
+
+static_always_inline u32
+vlib_buffer_enqueue_to_thread (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 frame_queue_index, u32 *buffer_indices,
+ u16 *thread_indices, u32 n_packets,
+ int drop_on_congestion)
+{
+ vlib_buffer_enqueue_to_thread_fn_t *fn;
+ fn = vlib_buffer_func_main.buffer_enqueue_to_thread_fn;
+ return (fn) (vm, node, frame_queue_index, buffer_indices, thread_indices,
+ n_packets, drop_on_congestion);
+}
+
#endif /* included_vlib_buffer_node_h */
/*