vlib: don't inline vlib_buffer_enqueue_to_*

[vpp.git] / src / vlib / buffer_node.h

diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h
index 8a77904..d2bd32f 100644 (file)
--- a/src/vlib/buffer_node.h
+++ b/src/vlib/buffer_node.h
@@ -69,6 +69,8 @@
 
 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
 do {                                                                   \
+  ASSERT (bi0 != 0);                                                   \
+  ASSERT (bi1 != 0);                                                   \
   int enqueue_code = (next0 != next_index) + 2*(next1 != next_index);  \
                                                                        \
   if (PREDICT_FALSE (enqueue_code != 0))                               \
@@ -137,9 +139,13 @@ do {                                                                       \
 
 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
 do {                                                                    \
+  ASSERT (bi0 != 0);                                                   \
+  ASSERT (bi1 != 0);                                                   \
+  ASSERT (bi2 != 0);                                                   \
+  ASSERT (bi3 != 0);                                                   \
   /* After the fact: check the [speculative] enqueue to "next" */       \
-  u32 fix_speculation = next_index != next0 || next_index != next1      \
-    || next_index != next2 || next_index != next3;                      \
+  u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1)     \
+    | (next_index ^ next2) | (next_index ^ next3);                      \
   if (PREDICT_FALSE(fix_speculation))                                   \
     {                                                                   \
       /* rewind... */                                                   \
@@ -181,15 +187,17 @@ do {                                                                    \
           n_left_to_next --;                                            \
         }                                                               \
       else                                                              \
-        vlib_set_next_frame_buffer (vm, node, next3, bi3);              \
-                                                                        \
-      /* Change speculation: last 2 packets went to the same node */    \
-      if (next2 == next3)                                               \
         {                                                               \
-          vlib_put_next_frame (vm, node, next_index, n_left_to_next);   \
-          next_index = next3;                                           \
-          vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
-        }                                                               \
+          vlib_set_next_frame_buffer (vm, node, next3, bi3);            \
+                                                                        \
+          /* Change speculation: last 2 packets went to the same node*/ \
+          if (next2 == next3)                                           \
+            {                                                           \
+              vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
+              next_index = next3;                                       \
+              vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
+            }                                                           \
+       }                                                               \
     }                                                                   \
  } while(0);
 
@@ -215,6 +223,7 @@ do {                                                                    \
 */
 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
 do {                                                                   \
+  ASSERT (bi0 != 0);                                                   \
   if (PREDICT_FALSE (next0 != next_index))                             \
     {                                                                  \
       vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);  \
@@ -326,6 +335,36 @@ generic_buffer_node_inline (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static_always_inline void
+vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
+                            u32 * buffers, u16 * nexts, uword count)
+{
+  vlib_buffer_enqueue_to_next_fn_t *fn;
+  fn = vlib_buffer_func_main.buffer_enqueue_to_next_fn;
+  (fn) (vm, node, buffers, nexts, count);
+}
+
+static_always_inline void
+vlib_buffer_enqueue_to_single_next (vlib_main_t * vm,
+                                   vlib_node_runtime_t * node, u32 * buffers,
+                                   u16 next_index, u32 count)
+{
+  vlib_buffer_enqueue_to_single_next_fn_t *fn;
+  fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_fn;
+  (fn) (vm, node, buffers, next_index, count);
+}
+
+static_always_inline u32
+vlib_buffer_enqueue_to_thread (vlib_main_t * vm, u32 frame_queue_index,
+                              u32 * buffer_indices, u16 * thread_indices,
+                              u32 n_packets, int drop_on_congestion)
+{
+  vlib_buffer_enqueue_to_thread_fn_t *fn;
+  fn = vlib_buffer_func_main.buffer_enqueue_to_thread_fn;
+  return (fn) (vm, frame_queue_index, buffer_indices, thread_indices,
+              n_packets, drop_on_congestion);
+}
+
 #endif /* included_vlib_buffer_node_h */
 
 /*