Break up vpe.api

[vpp.git] / src / vlib / buffer_funcs.h

diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 1aaac0b..c7b5321 100644 (file)
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -162,7 +162,10 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
 always_inline u64
 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
 {
-  return vlib_physmem_offset_to_physical (&vm->physmem_main,
+  vlib_physmem_region_index_t pri;
+  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
+  pri = vm->buffer_main->buffer_pools[b->buffer_pool_index].physmem_region;
+  return vlib_physmem_offset_to_physical (vm, pri,
                                          (((uword) buffer_index) <<
                                           CLIB_LOG2_CACHE_LINE_BYTES) +
                                          STRUCT_OFFSET_OF (vlib_buffer_t,
@@ -219,9 +222,10 @@ always_inline vlib_buffer_known_state_t
 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
-  ASSERT (vlib_get_thread_index () == 0);
 
+  clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   uword *p = hash_get (bm->buffer_known_hash, buffer_index);
+  clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
   return p ? p[0] : VLIB_BUFFER_UNKNOWN;
 }
 
@@ -231,8 +235,9 @@ vlib_buffer_set_known_state (vlib_main_t * vm,
                             vlib_buffer_known_state_t state)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
-  ASSERT (vlib_get_thread_index () == 0);
+  clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   hash_set (bm->buffer_known_hash, buffer_index, state);
+  clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
 }
 
 /* Validates sanity of a single buffer.
@@ -453,43 +458,6 @@ vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
     }
 }
 
-always_inline void *
-vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
-                           uword n_bytes, uword alignment)
-{
-  void *r =
-    vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
-  if (!r)
-    *error =
-      clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
-                        n_bytes);
-  else
-    *error = 0;
-  return r;
-}
-
-/* By default allocate I/O memory with cache line alignment. */
-always_inline void *
-vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
-{
-  return vlib_physmem_alloc_aligned (vm, error, n_bytes,
-                                    CLIB_CACHE_LINE_BYTES);
-}
-
-always_inline void
-vlib_physmem_free (vlib_main_t * vm, void *mem)
-{
-  return vm->os_physmem_free (mem);
-}
-
-always_inline u64
-vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem)
-{
-  vlib_physmem_main_t *pm = &vm->physmem_main;
-  uword o = pointer_to_uword (mem) - pm->virtual.start;
-  return vlib_physmem_offset_to_physical (pm, o);
-}
-
 /* Append given data to end of buffer, possibly allocating new buffers. */
 u32 vlib_buffer_add_data (vlib_main_t * vm,
                          u32 free_list_index,
@@ -831,7 +799,12 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
   _(current_length);
   _(flags);
 #undef _
-  ASSERT (dst->total_length_not_including_first_buffer == 0);
+  /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
+  /* total_length_not_including_first_buffer is not in the template anymore
+   * so it may actually not zeroed for some buffers. One option is to
+   * uncomment the line lower (comes at a cost), the other, is to just  not
+   * care */
+  /* dst->total_length_not_including_first_buffer = 0; */
   ASSERT (dst->n_add_refs == 0);
 }
 
@@ -845,6 +818,19 @@ vlib_buffer_add_to_free_list (vlib_main_t * vm,
   if (PREDICT_TRUE (do_init))
     vlib_buffer_init_for_free_list (b, f);
   vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
+
+  if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
+    {
+      vlib_buffer_free_list_t *mf;
+      mf = vlib_buffer_get_free_list (vlib_mains[0], f->index);
+      clib_spinlock_lock (&mf->global_buffers_lock);
+      /* keep last stored buffers, as they are more likely hot in the cache */
+      vec_add_aligned (mf->global_buffers, f->buffers, VLIB_FRAME_SIZE,
+                      CLIB_CACHE_LINE_BYTES);
+      vec_delete (f->buffers, VLIB_FRAME_SIZE, 0);
+      f->n_alloc -= VLIB_FRAME_SIZE;
+      clib_spinlock_unlock (&mf->global_buffers_lock);
+    }
 }
 
 always_inline void