vlib: don't inline vlib_buffer_enqueue_to_*

[vpp.git] / src / vlib / buffer_funcs.h

diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index a2d779b..e4427d6 100644 (file)
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -51,6 +51,26 @@
     vlib buffer access methods.
 */
 
+typedef void (vlib_buffer_enqueue_to_next_fn_t) (vlib_main_t *vm,
+                                                vlib_node_runtime_t *node,
+                                                u32 *buffers, u16 *nexts,
+                                                uword count);
+typedef void (vlib_buffer_enqueue_to_single_next_fn_t) (
+  vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index,
+  u32 count);
+
+typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) (
+  vlib_main_t *vm, u32 frame_queue_index, u32 *buffer_indices,
+  u16 *thread_indices, u32 n_packets, int drop_on_congestion);
+typedef struct
+{
+  vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn;
+  vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn;
+  vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn;
+} vlib_buffer_func_main_t;
+
+extern vlib_buffer_func_main_t vlib_buffer_func_main;
+
 always_inline void
 vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
 {
@@ -221,8 +241,8 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
      se we maintain two-in-parallel variant */
   while (count >= 8)
     {
-      u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
-      u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
+      u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
+      u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4));
       /* shift and add to get vlib_buffer_t pointer */
       u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
       u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
@@ -234,18 +254,18 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
   while (count >= 4)
     {
 #ifdef CLIB_HAVE_VEC256
-      u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
+      u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
       /* shift and add to get vlib_buffer_t pointer */
       u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
 #elif defined (CLIB_HAVE_VEC128)
       u64x2 off = u64x2_splat (buffer_mem_start + offset);
       u32x4 bi4 = u32x4_load_unaligned (bi);
-      u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
+      u64x2 b0 = u64x2_from_u32x4 ((u32x4) bi4);
 #if defined (__aarch64__)
-      u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4);
+      u64x2 b1 = u64x2_from_u32x4_high ((u32x4) bi4);
 #else
       bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
-      u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
+      u64x2 b1 = u64x2_from_u32x4 ((u32x4) bi4);
 #endif
       u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
       u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
@@ -786,28 +806,42 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
   vlib_buffer_t bt = { };
 #if defined(CLIB_HAVE_VEC128)
   vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
-  vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
+  vlib_buffer_t bpi_vec = {};
   vlib_buffer_t flags_refs_mask = {
     .flags = VLIB_BUFFER_NEXT_PRESENT,
     .ref_count = ~1
   };
 #endif
 
+  if (PREDICT_FALSE (n_buffers == 0))
+    return;
+
+  vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]);
+  buffer_pool_index = b->buffer_pool_index;
+  bp = vlib_get_buffer_pool (vm, buffer_pool_index);
+  vlib_buffer_copy_template (&bt, &bp->buffer_template);
+#if defined(CLIB_HAVE_VEC128)
+  bpi_vec.buffer_pool_index = buffer_pool_index;
+#endif
+
   while (n_buffers)
     {
       vlib_buffer_t *b[8];
       u32 bi, sum = 0, flags, next;
 
-      if (n_buffers < 12)
+      if (n_buffers < 4)
        goto one_by_one;
 
       vlib_get_buffers (vm, buffers, b, 4);
-      vlib_get_buffers (vm, buffers + 8, b + 4, 4);
 
-      vlib_prefetch_buffer_header (b[4], LOAD);
-      vlib_prefetch_buffer_header (b[5], LOAD);
-      vlib_prefetch_buffer_header (b[6], LOAD);
-      vlib_prefetch_buffer_header (b[7], LOAD);
+      if (n_buffers >= 12)
+       {
+         vlib_get_buffers (vm, buffers + 8, b + 4, 4);
+         vlib_prefetch_buffer_header (b[4], LOAD);
+         vlib_prefetch_buffer_header (b[5], LOAD);
+         vlib_prefetch_buffer_header (b[6], LOAD);
+         vlib_prefetch_buffer_header (b[7], LOAD);
+       }
 
 #if defined(CLIB_HAVE_VEC128)
       u8x16 p0, p1, p2, p3, r;