VCL: drain the vpp app event queue.

[vpp.git] / src / vlib / buffer_funcs.h

diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 7224e08..08f5b3e 100644 (file)
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -218,6 +218,11 @@ typedef enum
   VLIB_BUFFER_KNOWN_ALLOCATED,
 } vlib_buffer_known_state_t;
 
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+                                     uword n_buffers,
+                                     vlib_buffer_known_state_t
+                                     expected_state);
+
 always_inline vlib_buffer_known_state_t
 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
 {
@@ -245,24 +250,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm,
 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
                          uword follow_chain);
 
-/** \brief Allocate buffers into supplied array
-
-    @param vm - (vlib_main_t *) vlib main data structure pointer
-    @param buffers - (u32 * ) buffer index array
-    @param n_buffers - (u32) number of buffers requested
-    @return - (u32) number of buffers actually allocated, may be
-    less than the number requested or zero
-*/
-always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  ASSERT (bm->cb.vlib_buffer_alloc_cb);
-
-  return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
-}
-
 always_inline u32
 vlib_buffer_round_size (u32 size)
 {
@@ -301,11 +288,62 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
                                  u32 n_buffers, u32 free_list_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
+  vlib_buffer_free_list_t *fl;
+  u32 *src;
+  uword len;
+
+  ASSERT (bm->cb.vlib_buffer_fill_free_list_cb);
+
+  fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
+
+  len = vec_len (fl->buffers);
+
+  if (PREDICT_FALSE (len < n_buffers))
+    {
+      bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
+      len = vec_len (fl->buffers);
 
-  ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+      /* even if fill free list didn't manage to refill free list
+         we should give what we have */
+      n_buffers = clib_min (len, n_buffers);
 
-  return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
-                                                    free_list_index);
+      /* following code is intentionaly duplicated to allow compiler
+         to optimize fast path when n_buffers is constant value */
+      src = fl->buffers + len - n_buffers;
+      clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+      _vec_len (fl->buffers) -= n_buffers;
+
+      /* Verify that buffers are known free. */
+      vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+                                      VLIB_BUFFER_KNOWN_FREE);
+
+      return n_buffers;
+    }
+
+  src = fl->buffers + len - n_buffers;
+  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+  _vec_len (fl->buffers) -= n_buffers;
+
+  /* Verify that buffers are known free. */
+  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+                                  VLIB_BUFFER_KNOWN_FREE);
+
+  return n_buffers;
+}
+
+/** \brief Allocate buffers into supplied array
+
+    @param vm - (vlib_main_t *) vlib main data structure pointer
+    @param buffers - (u32 * ) buffer index array
+    @param n_buffers - (u32) number of buffers requested
+    @return - (u32) number of buffers actually allocated, may be
+    less than the number requested or zero
+*/
+always_inline u32
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+  return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+                                          VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 }
 
 /** \brief Free buffers
@@ -828,6 +866,7 @@ vlib_buffer_add_to_free_list (vlib_main_t * vm,
       vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE,
                       CLIB_CACHE_LINE_BYTES);
       vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+      f->n_alloc -= VLIB_FRAME_SIZE;
       clib_spinlock_unlock (&mf->global_buffers_lock);
     }
 }