vlib_node_runtime_t *node,
u32 *buffers, u16 *nexts,
uword count);
+typedef void (vlib_buffer_enqueue_to_next_with_aux_fn_t) (
+ vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u32 *aux_data,
+ u16 *nexts, uword count);
typedef void (vlib_buffer_enqueue_to_single_next_fn_t) (
vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index,
u32 count);
+typedef void (vlib_buffer_enqueue_to_single_next_with_aux_fn_t) (
+ vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u32 *aux_data,
+ u16 next_index, u32 count);
+
typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) (
vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index,
u32 *buffer_indices, u16 *thread_indices, u32 n_packets,
typedef struct
{
vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn;
+ vlib_buffer_enqueue_to_next_with_aux_fn_t
+ *buffer_enqueue_to_next_with_aux_fn;
vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn;
+ vlib_buffer_enqueue_to_single_next_with_aux_fn_t
+ *buffer_enqueue_to_single_next_with_aux_fn;
vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn;
vlib_buffer_enqueue_to_thread_with_aux_fn_t
*buffer_enqueue_to_thread_with_aux_fn;
}
}
-STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
static_always_inline void
vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
{
clib_spinlock_unlock (&bp->lock);
}
+/** \brief return unused buffers back to pool
+ This function can be used to return buffers back to pool without going
+ through vlib_buffer_free. Buffer metadata must not be modified in any
+ way before buffers are returned.
+
+ @param vm - (vlib_main_t *) vlib main data structure pointer
+ @param buffers - (u32 * ) buffer index array
+ @param n_buffers - (u32) number of buffers to free
+ @param buffer_pool_index - (u8) buffer pool index
+*/
+always_inline void
+vlib_buffer_unalloc_to_pool (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
+ u8 buffer_pool_index)
+{
+ vlib_buffer_pool_put (vm, buffer_pool_index, buffers, n_buffers);
+}
+
static_always_inline void
vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
int maybe_next)
vlib_buffer_pool_t *bp = 0;
u8 buffer_pool_index = ~0;
u32 n_queue = 0, queue[queue_size + 4];
- vlib_buffer_t bt = { };
+ vlib_buffer_template_t bt = {};
#if defined(CLIB_HAVE_VEC128)
vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
vlib_buffer_t bpi_vec = {};
vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]);
buffer_pool_index = b->buffer_pool_index;
bp = vlib_get_buffer_pool (vm, buffer_pool_index);
- vlib_buffer_copy_template (&bt, &bp->buffer_template);
+ bt = bp->buffer_template;
#if defined(CLIB_HAVE_VEC128)
bpi_vec.buffer_pool_index = buffer_pool_index;
#endif
vlib_buffer_t *b[8];
u32 bi, sum = 0, flags, next;
+#if defined(CLIB_HAVE_VEC512)
+ if (n_buffers < 8)
+#else
if (n_buffers < 4)
+#endif
goto one_by_one;
+#if defined(CLIB_HAVE_VEC512)
+ vlib_get_buffers (vm, buffers, b, 8);
+#else
vlib_get_buffers (vm, buffers, b, 4);
if (n_buffers >= 12)
vlib_prefetch_buffer_header (b[6], LOAD);
vlib_prefetch_buffer_header (b[7], LOAD);
}
+#endif
-#if defined(CLIB_HAVE_VEC128)
+#if defined(CLIB_HAVE_VEC512)
+ u8x16 p0, p1, p2, p3, p4, p5, p6, p7, r;
+ p0 = u8x16_load_unaligned (b[0]);
+ p1 = u8x16_load_unaligned (b[1]);
+ p2 = u8x16_load_unaligned (b[2]);
+ p3 = u8x16_load_unaligned (b[3]);
+ p4 = u8x16_load_unaligned (b[4]);
+ p5 = u8x16_load_unaligned (b[5]);
+ p6 = u8x16_load_unaligned (b[6]);
+ p7 = u8x16_load_unaligned (b[7]);
+
+ r = p0 ^ bpi_vec.as_u8x16[0];
+ r |= p1 ^ bpi_vec.as_u8x16[0];
+ r |= p2 ^ bpi_vec.as_u8x16[0];
+ r |= p3 ^ bpi_vec.as_u8x16[0];
+ r |= p4 ^ bpi_vec.as_u8x16[0];
+ r |= p5 ^ bpi_vec.as_u8x16[0];
+ r |= p6 ^ bpi_vec.as_u8x16[0];
+ r |= p7 ^ bpi_vec.as_u8x16[0];
+ r &= bpi_mask.as_u8x16[0];
+ r |=
+ (p0 | p1 | p2 | p3 | p4 | p5 | p6 | p7) & flags_refs_mask.as_u8x16[0];
+
+ sum = !u8x16_is_all_zero (r);
+#elif defined(CLIB_HAVE_VEC128)
u8x16 p0, p1, p2, p3, r;
p0 = u8x16_load_unaligned (b[0]);
p1 = u8x16_load_unaligned (b[1]);
if (sum)
goto one_by_one;
+#if defined(CLIB_HAVE_VEC512)
+ vlib_buffer_copy_indices (queue + n_queue, buffers, 8);
+ b[0]->template = bt;
+ b[1]->template = bt;
+ b[2]->template = bt;
+ b[3]->template = bt;
+ b[4]->template = bt;
+ b[5]->template = bt;
+ b[6]->template = bt;
+ b[7]->template = bt;
+ n_queue += 8;
+
+ vlib_buffer_validate (vm, b[0]);
+ vlib_buffer_validate (vm, b[1]);
+ vlib_buffer_validate (vm, b[2]);
+ vlib_buffer_validate (vm, b[3]);
+ vlib_buffer_validate (vm, b[4]);
+ vlib_buffer_validate (vm, b[5]);
+ vlib_buffer_validate (vm, b[6]);
+ vlib_buffer_validate (vm, b[7]);
+
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[4]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[5]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[6]);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[7]);
+#else
vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
- vlib_buffer_copy_template (b[0], &bt);
- vlib_buffer_copy_template (b[1], &bt);
- vlib_buffer_copy_template (b[2], &bt);
- vlib_buffer_copy_template (b[3], &bt);
+ b[0]->template = bt;
+ b[1]->template = bt;
+ b[2]->template = bt;
+ b[3]->template = bt;
n_queue += 4;
vlib_buffer_validate (vm, b[0]);
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
+#endif
if (n_queue >= queue_size)
{
vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
n_queue = 0;
}
+#if defined(CLIB_HAVE_VEC512)
+ buffers += 8;
+ n_buffers -= 8;
+#else
buffers += 4;
n_buffers -= 4;
+#endif
continue;
one_by_one:
bpi_vec.buffer_pool_index = buffer_pool_index;
#endif
bp = vlib_get_buffer_pool (vm, buffer_pool_index);
- vlib_buffer_copy_template (&bt, &bp->buffer_template);
+ bt = bp->buffer_template;
}
vlib_buffer_validate (vm, b[0]);
if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
{
- vlib_buffer_copy_template (b[0], &bt);
+ b[0]->template = bt;
queue[n_queue++] = bi;
}
Code Review / vpp.git / blobdiff