vlib: store buffer memory information in the buffer_main

Currently, buffer index is calculated as a offset to the physmem
region shifted by log2_cacheline size.

When DPDK is used we "hack" physmem data with information taken from
dpdk mempool. This makes physmem code not usable with DPDK.

This change makes buffer memory start and size independent of physmem
basically allowing physmem to be used when DPDK plugin is loaded.

Change-Id: Ieb399d398f147583b9baab467152a352d58c9c31
Signed-off-by: Damjan Marion <damarion@cisco.com>

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index fd1d841..aa73eb6 100644 (file)
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -431,7 +431,6 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
                         unsigned socket_id)
 {
   dpdk_main_t *dm = &dpdk_main;
-  vlib_physmem_main_t *vpm = &vm->physmem_main;
   struct rte_mempool *rmp;
   int i;
 
@@ -453,63 +452,10 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
   if (rmp)
     {
       {
-       uword this_pool_end;
-       uword this_pool_start;
-       uword this_pool_size;
-       uword save_vpm_start, save_vpm_end, save_vpm_size;
        struct rte_mempool_memhdr *memhdr;
 
-       this_pool_start = ~0;
-       this_pool_end = 0;
-
        STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
-       {
-         if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
-           this_pool_end = (uword) (memhdr->addr + memhdr->len);
-         if (((uword) memhdr->addr) < this_pool_start)
-           this_pool_start = (uword) (memhdr->addr);
-       }
-       ASSERT (this_pool_start < ~0 && this_pool_end > 0);
-       this_pool_size = this_pool_end - this_pool_start;
-
-       if (CLIB_DEBUG > 1)
-         {
-           clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
-                         pool_name, this_pool_start, this_pool_end,
-                         this_pool_size);
-           clib_warning
-             ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
-              vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-         }
-
-       save_vpm_start = vpm->virtual.start;
-       save_vpm_end = vpm->virtual.end;
-       save_vpm_size = vpm->virtual.size;
-
-       if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
-         vpm->virtual.start = this_pool_start;
-       if (this_pool_end > vpm->virtual.end)
-         vpm->virtual.end = this_pool_end;
-
-       vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
-
-       if (CLIB_DEBUG > 1)
-         {
-           clib_warning
-             ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
-              vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-         }
-
-       /* check if fits into buffer index range */
-       if ((u64) vpm->virtual.size >
-           ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
-         {
-           clib_warning ("physmem: virtual size out of range!");
-           vpm->virtual.start = save_vpm_start;
-           vpm->virtual.end = save_vpm_end;
-           vpm->virtual.size = save_vpm_size;
-           rmp = 0;
-         }
+         vlib_buffer_add_mem_range (vm, (uword) memhdr->addr, memhdr->len);
       }
       if (rmp)
        {
@@ -564,7 +510,8 @@ buffer_state_validation_init (vlib_main_t * vm)
 VLIB_INIT_FUNCTION (buffer_state_validation_init);
 #endif
 
-static vlib_buffer_callbacks_t callbacks = {
+/* *INDENT-OFF* */
+VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
   .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
   .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
   .vlib_buffer_free_cb = &dpdk_buffer_free,
@@ -572,15 +519,7 @@ static vlib_buffer_callbacks_t callbacks = {
   .vlib_packet_template_init_cb = &dpdk_packet_template_init,
   .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
 };
-
-static clib_error_t *
-dpdk_buffer_init (vlib_main_t * vm)
-{
-  vlib_buffer_cb_register (vm, &callbacks);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_buffer_init);
+/* *INDENT-ON* */
 
 /** @endcond */
 /*

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index e50064a..b2a095c 100644 (file)
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -46,6 +46,8 @@
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
 
+vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
+
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
                                       vlib_buffer_t * b_first)
@@ -584,11 +586,6 @@ alloc_from_free_list (vlib_main_t * vm,
 
   dst = alloc_buffers;
 
-  /* wait with buffer memory allocation as long as possible
-     in case external buffer manager takes over */
-  if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0))
-    unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
-
   n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
   if (n_filled == 0)
     return 0;
@@ -944,6 +941,36 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
   return copied;
 }
 
+void
+vlib_buffer_add_mem_range (vlib_main_t * vm, uword start, uword size)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+
+  if (bm->buffer_mem_size == 0)
+    {
+      bm->buffer_mem_start = start;
+      bm->buffer_mem_size = size;
+    }
+  else if (start < bm->buffer_mem_start)
+    {
+      bm->buffer_mem_size += bm->buffer_mem_start - start;
+      bm->buffer_mem_start = start;
+      if (size > bm->buffer_mem_size)
+       bm->buffer_mem_size = size;
+    }
+  else if (start > bm->buffer_mem_start)
+    {
+      uword new_size = start - bm->buffer_mem_start + size;
+      if (new_size > bm->buffer_mem_size)
+       bm->buffer_mem_size = new_size;
+    }
+
+  if ((u64) bm->buffer_mem_size >
+      ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
+    {
+      clib_panic ("buffer memory size out of range!");
+    }
+}
 
 static u8 *
 format_vlib_buffer_free_list (u8 * s, va_list * va)
@@ -1011,6 +1038,7 @@ void
 vlib_buffer_cb_init (struct vlib_main_t *vm)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
+
   bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal;
   bm->cb.vlib_buffer_alloc_from_free_list_cb =
     &vlib_buffer_alloc_from_free_list_internal;
@@ -1018,25 +1046,6 @@ vlib_buffer_cb_init (struct vlib_main_t *vm)
   bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
   bm->cb.vlib_buffer_delete_free_list_cb =
     &vlib_buffer_delete_free_list_internal;
-  bm->extern_buffer_mgmt = 0;
-}
-
-int
-vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  if (bm->extern_buffer_mgmt)
-    return -1;
-
-#define _(x) bm->cb.x = cb->x
-  _(vlib_buffer_alloc_cb);
-  _(vlib_buffer_alloc_from_free_list_cb);
-  _(vlib_buffer_free_cb);
-  _(vlib_buffer_free_no_next_cb);
-  _(vlib_buffer_delete_free_list_cb);
-#undef _
-  bm->extern_buffer_mgmt = 1;
-  return 0;
 }
 
 /** @endcond */

diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 18e2437..b20538b 100644 (file)
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -388,12 +388,20 @@ typedef struct
                                           u32 free_list_index);
 } vlib_buffer_callbacks_t;
 
+extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
+
 typedef struct
 {
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  /* Virtual memory address and size of buffer memory, used for calculating
+     buffer index */
+  uword buffer_mem_start;
+  uword buffer_mem_size;
+
   /* Buffer free callback, for subversive activities */
-  u32 (*buffer_free_callback) (struct vlib_main_t * vm,
-                              u32 * buffers,
-                              u32 n_buffers, u32 follow_buffer_next);
+    u32 (*buffer_free_callback) (struct vlib_main_t * vm,
+                                u32 * buffers,
+                                u32 n_buffers, u32 follow_buffer_next);
   /* Pool of buffer free lists.
      Multiple free lists exist for packet generator which uses
      separate free lists for each packet stream --- so as to avoid
@@ -417,12 +425,12 @@ typedef struct
 
   /* Callbacks */
   vlib_buffer_callbacks_t cb;
-  int extern_buffer_mgmt;
+  int callbacks_registered;
 } vlib_buffer_main_t;
 
+void vlib_buffer_add_mem_range (struct vlib_main_t *vm, uword start,
+                               uword size);
 void vlib_buffer_cb_init (struct vlib_main_t *vm);
-int vlib_buffer_cb_register (struct vlib_main_t *vm,
-                            vlib_buffer_callbacks_t * cb);
 
 typedef struct
 {
@@ -498,6 +506,18 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m)
 
 #endif /* included_vlib_buffer_h */
 
+#define VLIB_BUFFER_REGISTER_CALLBACKS(x,...)                           \
+    __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks;       \
+static void __vlib_add_buffer_callbacks_t_##x (void)                    \
+    __attribute__((__constructor__)) ;                                  \
+static void __vlib_add_buffer_callbacks_t_##x (void)                    \
+{                                                                       \
+    if (vlib_buffer_callbacks)                                          \
+      clib_panic ("vlib buffer callbacks already registered");          \
+    vlib_buffer_callbacks = &__##x##_buffer_callbacks;                  \
+}                                                                       \
+__VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
+
 /*
  * fd.io coding-style-patch-verification: ON
  *

diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 328660a..79e3e69 100644 (file)
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -56,8 +56,11 @@
 always_inline vlib_buffer_t *
 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
 {
-  return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
-                                << CLIB_LOG2_CACHE_LINE_BYTES);
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+  ASSERT (offset < bm->buffer_mem_size);
+
+  return uword_to_pointer (bm->buffer_mem_start + offset, void *);
 }
 
 /** \brief Translate buffer pointer into buffer index
@@ -66,10 +69,14 @@ vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
     @param p - (void *) buffer pointer
     @return - (u32) buffer index
 */
+
 always_inline u32
 vlib_get_buffer_index (vlib_main_t * vm, void *p)
 {
-  uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+  ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+  ASSERT (offset < bm->buffer_mem_size);
   ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
   return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
 }

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 19d7023..73548fb 100644 (file)
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -43,6 +43,7 @@
 #include <vlib/threads.h>
 #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
 
+#include <vlib/unix/unix.h>
 #include <vlib/unix/cj.h>
 
 CJ_GLOBAL_LOG_PROTOTYPE;
@@ -466,7 +467,7 @@ vlib_put_next_frame (vlib_main_t * vm,
   vlib_frame_t *f;
   u32 n_vectors_in_frame;
 
-  if (vm->buffer_main->extern_buffer_mgmt == 0 && CLIB_DEBUG > 0)
+  if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0)
     vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
 
   nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
@@ -1712,7 +1713,22 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
     vm->name = "VLIB";
 
   vec_validate (vm->buffer_main, 0);
-  vlib_buffer_cb_init (vm);
+  if (vlib_buffer_callbacks)
+    {
+      /* external plugin has registered own buffer callbacks
+         so we just copy them */
+      vlib_buffer_main_t *bm = vm->buffer_main;
+      clib_memcpy (&bm->cb, vlib_buffer_callbacks,
+                  sizeof (vlib_buffer_callbacks_t));
+      bm->callbacks_registered = 1;
+    }
+  else
+    {
+      vlib_physmem_main_t *vpm = &vm->physmem_main;
+      vlib_buffer_cb_init (vm);
+      unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
+      vlib_buffer_add_mem_range (vm, vpm->virtual.start, vpm->virtual.size);
+    }
 
   if ((error = vlib_thread_init (vm)))
     {

diff --git a/src/vlib/unix/physmem.c b/src/vlib/unix/physmem.c
index 8d10ad2..933cc63 100644 (file)
--- a/src/vlib/unix/physmem.c
+++ b/src/vlib/unix/physmem.c
@@ -45,14 +45,10 @@ static void *
 unix_physmem_alloc_aligned (vlib_physmem_main_t * vpm, uword n_bytes,
                            uword alignment)
 {
-  vlib_main_t *vm = vlib_get_main ();
   physmem_main_t *pm = &physmem_main;
   uword lo_offset, hi_offset;
   uword *to_free = 0;
 
-  if (vm->buffer_main->extern_buffer_mgmt)
-    clib_warning ("unsafe alloc!");
-
   /* IO memory is always at least cache aligned. */
   alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
 
@@ -270,11 +266,6 @@ show_physmem (vlib_main_t * vm,
              unformat_input_t * input, vlib_cli_command_t * cmd)
 {
   physmem_main_t *pm = &physmem_main;
-  if (vm->buffer_main->extern_buffer_mgmt)
-    {
-      vlib_cli_output (vm, "Not supported with external buffer management.");
-      return 0;
-    }
 
   if (pm->heap)
     vlib_cli_output (vm, "%U", format_mheap, pm->heap, /* verbose */ 1);

diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 597ae06..c3738a6 100644 (file)
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1214,7 +1214,7 @@ pg_stream_fill_helper (pg_main_t * pg,
    * Historically, the pg maintained its own free lists and
    * device drivers tx paths would return pkts.
    */
-  if (vm->buffer_main->extern_buffer_mgmt == 0 &&
+  if (vm->buffer_main->callbacks_registered == 0 &&
       !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     f->buffer_init_function = pg_buffer_init;
   f->buffer_init_function_opaque =
@@ -1238,7 +1238,7 @@ pg_stream_fill_helper (pg_main_t * pg,
   n_alloc = n_allocated;
 
   /* Reinitialize buffers */
-  if (vm->buffer_main->extern_buffer_mgmt == 0 || CLIB_DEBUG > 0
+  if (vm->buffer_main->callbacks_registered == 0 || CLIB_DEBUG > 0
       || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     init_buffers_inline
       (vm, s,
@@ -1246,7 +1246,7 @@ pg_stream_fill_helper (pg_main_t * pg,
        n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
        s->buffer_bytes,
        /* set_data */
-       vm->buffer_main->extern_buffer_mgmt != 0
+       vm->buffer_main->callbacks_registered != 0
        || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
 
   if (next_buffers)

diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index 05d820a..a540b32 100644 (file)
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -438,7 +438,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
     pg_buffer_index_t *bi;
     int n;
 
-    if (vm->buffer_main->extern_buffer_mgmt)
+    if (vm->buffer_main->callbacks_registered)
       s->buffer_bytes = VLIB_BUFFER_DATA_SIZE;
 
     if (!s->buffer_bytes)

diff --git a/src/vnet/replication.c b/src/vnet/replication.c
index 233a8c2..1c6f28d 100644 (file)
--- a/src/vnet/replication.c
+++ b/src/vnet/replication.c
@@ -214,7 +214,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
          b0->flags |= VLIB_BUFFER_IS_RECYCLED;
 
 #if (CLIB_DEBUG > 0)
-         if (vm->buffer_main->extern_buffer_mgmt == 0)
+         if (vm->buffer_main->callbacks_registered == 0)
            vlib_buffer_set_known_state (vm, bi0,
                                         VLIB_BUFFER_KNOWN_ALLOCATED);
 #endif