svm: remove fifo segment heap

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I518e096fe13847759806ff62009e73fd8f7451b7

diff --git a/src/plugins/unittest/segment_manager_test.c b/src/plugins/unittest/segment_manager_test.c
index c3e7e26..2e8261e 100644 (file)
--- a/src/plugins/unittest/segment_manager_test.c
+++ b/src/plugins/unittest/segment_manager_test.c
@@ -168,7 +168,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_enqueue (tx_fifo, fifo_size - 1, data);
 
   /* 256KB+ / 2048KB+ => ~12% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -182,7 +181,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_enqueue (tx_fifo, fifo_size, data);
 
   /* 8 chunks : 49% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -192,7 +190,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_enqueue (tx_fifo, fifo_size, data);
 
   /* 10 chunks : 61% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_LOW_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -204,7 +201,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_enqueue (tx_fifo, fifo_size, data);
 
   /* 14 chunks : 85% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_HIGH_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -217,7 +213,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_dequeue_drop (tx_fifo, fifo_size);
 
   /* 10 chunks : 61% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_LOW_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -230,7 +225,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_enqueue (tx_fifo, fifo_size, data);
 
   /* 14 chunks : 85% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_HIGH_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -242,7 +236,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
 
 
   /* 10 chunks : 61% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_LOW_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -258,7 +251,6 @@ segment_manager_test_pressure_1 (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_dequeue_drop (tx_fifo, fifo_size);
 
   /* 2 chunks : 12% */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -310,7 +302,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
 
   /* initial status : (0 / 2MB) */
   fs0 = segment_manager_get_segment (sm, 0);
-  fifo_segment_update_free_bytes (fs0);
   rv = fifo_segment_get_mem_status (fs0);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -338,7 +329,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
     }
 
   /* 510 chunks : 100% of 2MB */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_HIGH_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -348,7 +338,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
   SEG_MGR_TEST ((rv == SVM_FIFO_EGROW), "svm_fifo_enqueue %d", rv);
 
   /* then, no-memory is detected */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_MEMORY),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -363,7 +352,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
    * but the reached-mem-limit record is not reset
    * so the no-memory state lasts.
    */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_MEMORY),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -375,7 +363,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
     }
 
   /* 356 chunks : 70% of 2MB */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_LOW_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);
@@ -387,7 +374,6 @@ segment_manager_test_pressure_2 (vlib_main_t * vm, unformat_input_t * input)
     }
 
   /* 2 chunks : 3% of 2MB */
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_get_mem_status (fs);
   SEG_MGR_TEST ((rv == MEMORY_PRESSURE_NO_PRESSURE),
                "fifo_segment_get_mem_status %s", states_str[rv]);

diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c
index a3f88d3..a223e13 100644 (file)
--- a/src/plugins/unittest/svm_fifo_test.c
+++ b/src/plugins/unittest/svm_fifo_test.c
@@ -2535,6 +2535,14 @@ sfifo_test_fifo_segment_mempig (int verbose)
   return 0;
 }
 
+static int
+approx_leq (uword a, uword b, u32 margin)
+{
+  if (a - margin <= b && b <= a)
+    return 1;
+  return 0;
+}
+
 static int
 sfifo_test_fifo_segment_prealloc (int verbose)
 {
@@ -2568,8 +2576,10 @@ sfifo_test_fifo_segment_prealloc (int verbose)
   SFIFO_TEST (rv == 50, "prealloc chunks expected %u is %u", 50, rv);
   rv = fifo_segment_free_bytes (fs);
   free_space -= (sizeof (svm_fifo_chunk_t) + 4096) * 50;
-  SFIFO_TEST (rv == free_space, "free space expected %u is %u", free_space,
-             rv);
+  /* Memory alloc alignment accounts for the difference */
+  SFIFO_TEST (approx_leq (free_space, rv, 16), "free space expected %u is %u",
+             free_space, rv);
+  free_space = rv;
   rv = fifo_segment_fl_chunk_bytes (fs);
   SFIFO_TEST (rv == 4096 * 50, "chunk free space expected %u is %u",
              4096 * 50, rv);
@@ -2580,10 +2590,11 @@ sfifo_test_fifo_segment_prealloc (int verbose)
   SFIFO_TEST (rv == 50, "prealloc fifo hdrs expected %u is %u", 50, rv);
   rv = fifo_segment_free_bytes (fs);
   free_space -= sizeof (svm_fifo_t) * 50;
-  SFIFO_TEST (rv == free_space, "free space expected %u is %u", free_space,
-             rv);
+  /* Memory alloc alignment accounts for the difference */
+  SFIFO_TEST (approx_leq (free_space, rv, 16), "free space expected %u is %u",
+             free_space, rv);
+  free_space = rv;
 
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_free_bytes (fs);
   SFIFO_TEST (clib_abs (rv - (int) free_space) < 512,
              "free space expected %u is %u", free_space, rv);
@@ -2606,7 +2617,6 @@ sfifo_test_fifo_segment_prealloc (int verbose)
   /*
    * Multiple preallocs that consume the remaining space
    */
-  fifo_segment_update_free_bytes (fs);
   free_space = fifo_segment_free_bytes (fs);
   pair_mem = 2 * (4096 + sizeof (*f) + sizeof (svm_fifo_chunk_t));
   max_pairs = pairs_req = (free_space / pair_mem) - 1;
@@ -2616,7 +2626,6 @@ sfifo_test_fifo_segment_prealloc (int verbose)
   SFIFO_TEST (rv == max_pairs * 2, "prealloc chunks expected %u is %u",
              max_pairs * 2, rv);
 
-  fifo_segment_update_free_bytes (fs);
   rv = fifo_segment_free_bytes (fs);
   SFIFO_TEST (rv < 2 * pair_mem, "free bytes %u less than %u", rv,
              2 * pair_mem);

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index e25967e..d474b85 100644 (file)
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -15,6 +15,36 @@
 
 #include <svm/fifo_segment.h>
 
+static inline void *
+fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align)
+{
+  uword cur_pos, cur_pos_align, new_pos;
+
+  cur_pos = clib_atomic_load_relax_n (&fsh->byte_index);
+  cur_pos_align = round_pow2_u64 (cur_pos, align);
+  size = round_pow2_u64 (size, align);
+  new_pos = cur_pos_align + size;
+
+  if (new_pos >= fsh->max_byte_index)
+    return 0;
+
+  while (!clib_atomic_cmp_and_swap_acq_relax (&fsh->byte_index, &cur_pos,
+                                             &new_pos, 1 /* weak */))
+    {
+      cur_pos_align = round_pow2_u64 (cur_pos, align);
+      new_pos = cur_pos_align + size;
+      if (new_pos >= fsh->max_byte_index)
+       return 0;
+    }
+  return uword_to_pointer ((u8 *) fsh + cur_pos_align, void *);
+}
+
+static inline void *
+fsh_alloc (fifo_segment_header_t *fsh, uword size)
+{
+  return fsh_alloc_aligned (fsh, size, 8);
+}
+
 static inline fifo_segment_slice_t *
 fsh_slice_get (fifo_segment_header_t * fsh, u32 slice_index)
 {
@@ -27,39 +57,12 @@ static char *fifo_segment_mem_status_strings[] = {
 #undef _
 };
 
-/**
- * Fifo segment free space
- *
- * Queries the underlying memory manager, dlmalloc, for free space. Since this
- * ends up walking the internal data structures, it should not be called
- * indiscriminately.
- *
- * @param fs           fifo segment
- * @return             number of free bytes
- */
-static uword
-fsh_free_space (fifo_segment_header_t * fsh)
-{
-  return clib_mem_get_heap_free_space (fsh->ssvm_sh->heap);
-}
-
-static inline void
-fsh_free_bytes_sub (fifo_segment_header_t * fsh, int size)
-{
-  clib_atomic_fetch_sub_rel (&fsh->n_free_bytes, size);
-}
-
 static inline uword
 fsh_n_free_bytes (fifo_segment_header_t * fsh)
 {
-  uword n_free = clib_atomic_load_relax_n (&fsh->n_free_bytes);
-  return n_free > fsh->n_reserved_bytes ? n_free - fsh->n_reserved_bytes : 0;
-}
-
-static inline void
-fsh_update_free_bytes (fifo_segment_header_t * fsh)
-{
-  clib_atomic_store_rel_n (&fsh->n_free_bytes, fsh_free_space (fsh));
+  uword cur_pos = clib_atomic_load_relax_n (&fsh->byte_index);
+  ASSERT (fsh->max_byte_index > cur_pos);
+  return fsh->max_byte_index - cur_pos;
 }
 
 static inline void
@@ -116,37 +119,41 @@ fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
   fss->virtual_mem += n_bytes;
 }
 
-static void
-fsh_check_mem (fifo_segment_header_t * fsh)
+static inline void
+fss_chunk_freelist_lock (fifo_segment_slice_t *fss)
 {
-  uword thresh;
-
-  if (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT)
-    return;
-
-  thresh = clib_max (0.01 * fsh->ssvm_sh->ssvm_size,
-                    2 * fsh->n_reserved_bytes);
-  if (fsh->n_free_bytes > thresh)
-    return;
+  u32 free = 0;
+  while (!clib_atomic_cmp_and_swap_acq_relax_n (&fss->chunk_lock, &free, 1, 0))
+    {
+      /* atomic load limits number of compare_exchange executions */
+      while (clib_atomic_load_relax_n (&fss->chunk_lock))
+       CLIB_PAUSE ();
+      /* on failure, compare_exchange writes (*p)->lock into free */
+      free = 0;
+    }
+}
 
-  fsh->flags |= FIFO_SEGMENT_F_MEM_LIMIT;
-  fsh_update_free_bytes (fsh);
+static inline void
+fss_chunk_freelist_unlock (fifo_segment_slice_t *fss)
+{
+  /* Make sure all reads/writes are complete before releasing the lock */
+  clib_atomic_release (&fss->chunk_lock);
 }
 
 static inline int
 fss_chunk_fl_index_is_valid (fifo_segment_slice_t * fss, u32 fl_index)
 {
-  return (fl_index < vec_len (fss->free_chunks));
+  return (fl_index < fss->n_chunk_lens);
 }
 
 static void
 fss_chunk_free_list_push (fifo_segment_slice_t * fss, u32 fl_index,
                          svm_fifo_chunk_t * c)
 {
-  clib_spinlock_lock (&fss->chunk_lock);
+  fss_chunk_freelist_lock (fss);
   c->next = fss->free_chunks[fl_index];
   fss->free_chunks[fl_index] = c;
-  clib_spinlock_unlock (&fss->chunk_lock);
+  fss_chunk_freelist_unlock (fss);
 }
 
 static void
@@ -154,10 +161,10 @@ fss_chunk_free_list_push_list (fifo_segment_slice_t * fss, u32 fl_index,
                               svm_fifo_chunk_t * head,
                               svm_fifo_chunk_t * tail)
 {
-  clib_spinlock_lock (&fss->chunk_lock);
+  fss_chunk_freelist_lock (fss);
   tail->next = fss->free_chunks[fl_index];
   fss->free_chunks[fl_index] = head;
-  clib_spinlock_unlock (&fss->chunk_lock);
+  fss_chunk_freelist_unlock (fss);
 }
 
 static svm_fifo_chunk_t *
@@ -167,18 +174,18 @@ fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index)
 
   ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index));
 
-  clib_spinlock_lock (&fss->chunk_lock);
+  fss_chunk_freelist_lock (fss);
 
   if (!fss->free_chunks[fl_index])
     {
-      clib_spinlock_unlock (&fss->chunk_lock);
+      fss_chunk_freelist_unlock (fss);
       return 0;
     }
 
   c = fss->free_chunks[fl_index];
   fss->free_chunks[fl_index] = c->next;
 
-  clib_spinlock_unlock (&fss->chunk_lock);
+  fss_chunk_freelist_unlock (fss);
 
   return c;
 }
@@ -232,54 +239,57 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size)
 int
 fifo_segment_init (fifo_segment_t * fs)
 {
+  u32 max_chunk_sz, align = 8, offset = 2 * 4096, vec_sz;
+  uword max_fifo, seg_start, seg_size;
   fifo_segment_header_t *fsh;
   fifo_segment_slice_t *fss;
   ssvm_shared_header_t *sh;
-  u32 max_chunk_sz;
-  uword max_fifo;
-  void *oldheap;
+  void *seg_data;
   int i;
 
+  /* TODO remove ssvm heap entirely */
   sh = fs->ssvm.sh;
-  oldheap = ssvm_push_heap (sh);
 
-  /*
-   * Manually align the fifo segment header to sizeof(uword) = 8 bytes.
-   * Long story made short: the "process-private" fifo segment
-   * is allocated from the main heap, not mmapped. dlmalloc
-   * only guarantees 4-byte alignment, and on aarch64
-   * the fsh can end up 4-byte but not 8-byte aligned.
-   * That eventually causes the atomic op in fifo_segment_update_free_bytes
-   * to backfire.
-   */
-  fsh = clib_mem_alloc_aligned (sizeof (*fsh), sizeof (uword));
-  clib_memset (fsh, 0, sizeof (*fsh));
+  seg_data = (u8 *) sh + offset;
+  seg_size = sh->ssvm_size - offset;
+
+  seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align);
+  fsh = uword_to_pointer (seg_start, void *);
+  memset (fsh, 0, sizeof (*fsh));
+  fsh->byte_index = sizeof (*fsh);
+  fsh->max_byte_index = seg_size;
+  fs->max_byte_index = fsh->max_byte_index;
+  ASSERT (fsh->max_byte_index <= sh->ssvm_va + sh->ssvm_size);
+
   fs->h = sh->opaque[0] = fsh;
   fs->n_slices = clib_max (fs->n_slices, 1);
 
-  fsh->ssvm_sh = fs->ssvm.sh;
   fsh->n_slices = fs->n_slices;
-  max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2,
-                      FIFO_SEGMENT_MAX_FIFO_SIZE);
+  max_fifo =
+    clib_min ((fsh_n_free_bytes (fsh) - 4096) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE);
   fsh->max_log2_chunk_size = max_log2 (max_fifo);
 
-  fsh->slices = clib_mem_alloc (sizeof (*fss) * fs->n_slices);
-  clib_memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices);
+  fsh->slices = fsh_alloc (fsh, sizeof (*fss) * fs->n_slices);
+  memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices);
   max_chunk_sz = fsh->max_log2_chunk_size - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE;
+  max_chunk_sz += 1;
 
   for (i = 0; i < fs->n_slices; i++)
     {
       fss = fsh_slice_get (fsh, i);
-      vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0);
-      vec_validate_init_empty (fss->num_chunks, max_chunk_sz, 0);
-      clib_spinlock_init (&fss->chunk_lock);
+      vec_sz = max_chunk_sz * sizeof (*fss->free_chunks);
+      fss->free_chunks = fsh_alloc (fsh, vec_sz);
+      memset (fss->free_chunks, 0, vec_sz);
+      vec_sz = max_chunk_sz * sizeof (*fss->num_chunks);
+      fss->num_chunks = fsh_alloc (fsh, vec_sz);
+      memset (fss->num_chunks, 0, vec_sz);
+
+      fss->chunk_lock = 0;
+      fss->n_chunk_lens = max_chunk_sz;
     }
 
-  ssvm_pop_heap (oldheap);
-
-  fsh->n_free_bytes = fsh_free_space (fsh);
   fsh->n_cached_bytes = 0;
-  fsh->n_reserved_bytes = clib_min (0.01 * fsh->n_free_bytes, 256 << 10);
+  fsh->n_reserved_bytes = fsh->byte_index;
   sh->ready = 1;
   return (0);
 }
@@ -480,7 +490,6 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
                              fifo_segment_slice_t * fss, u32 batch_size)
 {
   svm_fifo_t *f;
-  void *oldheap;
   uword size;
   u8 *fmem;
   int i;
@@ -489,13 +498,7 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
 
   size = (uword) sizeof (*f) * batch_size;
 
-  oldheap = ssvm_push_heap (fsh->ssvm_sh);
-  fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES,
-                                          0 /* align_offset */ ,
-                                          0 /* os_out_of_memory */ );
-  ssvm_pop_heap (oldheap);
-
-  /* Out of space.. */
+  fmem = fsh_alloc_aligned (fsh, size, CLIB_CACHE_LINE_BYTES);
   if (fmem == 0)
     return -1;
 
@@ -509,8 +512,6 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
       fmem += sizeof (*f);
     }
 
-  fsh_free_bytes_sub (fsh, size);
-
   return 0;
 }
 
@@ -522,7 +523,6 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh,
   svm_fifo_chunk_t *c, *head = 0, *tail;
   uword size, total_chunk_bytes;
   u32 rounded_data_size;
-  void *oldheap;
   u8 *cmem;
   int i;
 
@@ -532,13 +532,7 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh,
   total_chunk_bytes = (uword) batch_size *rounded_data_size;
   size = (uword) (sizeof (*c) + rounded_data_size) * batch_size;
 
-  oldheap = ssvm_push_heap (fsh->ssvm_sh);
-  cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES,
-                                          0 /* align_offset */ ,
-                                          0 /* os_out_of_memory */ );
-  ssvm_pop_heap (oldheap);
-
-  /* Out of space.. */
+  cmem = fsh_alloc_aligned (fsh, size, 8 /* chunk hdr is 24B */);
   if (cmem == 0)
     return -1;
 
@@ -558,7 +552,6 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh,
   fss->num_chunks[fl_index] += batch_size;
   fss_fl_chunk_bytes_add (fss, total_chunk_bytes);
   fsh_cached_bytes_add (fsh, total_chunk_bytes);
-  fsh_free_bytes_sub (fsh, size);
 
   return 0;
 }
@@ -632,10 +625,7 @@ free_list:
            goto done;
          batch = n_free / FIFO_SEGMENT_MIN_FIFO_SIZE;
          if (!batch || fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
-           {
-             fsh_check_mem (fsh);
-             goto done;
-           }
+           goto done;
        }
       if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
        {
@@ -644,10 +634,7 @@ free_list:
          batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
          batch = clib_min (batch + 1, n_free / min_size);
          if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
-           {
-             fsh_check_mem (fsh);
-             goto done;
-           }
+           goto done;
          c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
        }
     }
@@ -952,7 +939,7 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
 
   /* Calculate space requirements */
   pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
-  space_available = fsh_free_space (fsh);
+  space_available = fsh_n_free_bytes (fsh);
   pairs_to_alloc = space_available / pair_size;
   pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs);
   pairs_per_slice = pairs_to_alloc / fs->n_slices;
@@ -1031,7 +1018,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
   /* Count all free chunks? */
   if (size == ~0)
     {
-      for (i = 0; i < vec_len (fss->free_chunks); i++)
+      for (i = 0; i < fss->n_chunk_lens; i++)
        {
          c = fss->free_chunks[i];
          if (c == 0)
@@ -1049,7 +1036,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
   rounded_size = (1 << (max_log2 (size)));
   fl_index = fs_freelist_for_size (rounded_size);
 
-  if (fl_index >= vec_len (fss->free_chunks))
+  if (fl_index >= fss->n_chunk_lens)
     return 0;
 
   c = fss->free_chunks[fl_index];
@@ -1080,16 +1067,10 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
   return count;
 }
 
-void
-fifo_segment_update_free_bytes (fifo_segment_t * fs)
-{
-  fsh_update_free_bytes (fs->h);
-}
-
 uword
 fifo_segment_size (fifo_segment_t * fs)
 {
-  return fs->ssvm.ssvm_size;
+  return fs->h->max_byte_index - fs->h->n_reserved_bytes;
 }
 
 u8
@@ -1104,6 +1085,16 @@ fsh_reset_mem_limit (fifo_segment_header_t * fsh)
   fsh->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT;
 }
 
+void *
+fifo_segment_alloc (fifo_segment_t *fs, uword size)
+{
+  void *rv = fsh_alloc (fs->h, size);
+  /* Mark externally allocated bytes as reserved. This helps
+   * @ref fifo_segment_size report bytes used only for fifos */
+  fs->h->n_reserved_bytes += size;
+  return rv;
+}
+
 uword
 fifo_segment_free_bytes (fifo_segment_t * fs)
 {
@@ -1276,7 +1267,7 @@ format_fifo_segment (u8 * s, va_list * args)
   for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
     {
       fss = fsh_slice_get (fsh, slice_index);
-      for (i = 0; i < vec_len (fss->free_chunks); i++)
+      for (i = 0; i < fss->n_chunk_lens; i++)
        {
          c = fss->free_chunks[i];
          if (c == 0 && fss->num_chunks[i] == 0)
@@ -1299,7 +1290,6 @@ format_fifo_segment (u8 * s, va_list * args)
   fifo_hdr = free_fifos * sizeof (svm_fifo_t);
   est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs);
   est_free_seg_bytes = fifo_segment_free_bytes (fs);
-  fifo_segment_update_free_bytes (fs);
   free_seg_bytes = fifo_segment_free_bytes (fs);
   tracked_cached_bytes = fifo_segment_cached_bytes (fs);
   allocated = fifo_segment_size (fs);

diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h
index d3ce59c..39c9454 100644 (file)
--- a/src/svm/fifo_segment.h
+++ b/src/svm/fifo_segment.h
@@ -67,6 +67,7 @@ typedef struct
 {
   ssvm_private_t ssvm;         /**< ssvm segment data */
   fifo_segment_header_t *h;    /**< fifo segment data */
+  uword max_byte_index;
   u8 n_slices;                 /**< number of fifo segment slices */
 } fifo_segment_t;
 
@@ -205,6 +206,14 @@ u8 fsh_has_reached_mem_limit (fifo_segment_header_t * fsh);
  */
 void fsh_reset_mem_limit (fifo_segment_header_t * fsh);
 
+/**
+ * Fifo segment reset mem limit flag
+ *
+ * @param fs            fifo segment
+ * @param size         size requested
+ * @return             pointer to memory allocated or 0
+ */
+void *fifo_segment_alloc (fifo_segment_t *fs, uword size);
 /**
  * Fifo segment allocated size
  *
@@ -227,16 +236,6 @@ uword fifo_segment_size (fifo_segment_t * fs);
  */
 uword fifo_segment_free_bytes (fifo_segment_t * fs);
 
-/**
- * Update fifo segment free bytes estimate
- *
- * Forces fifo segment free bytes estimate synchronization with underlying
- * memory allocator.
- *
- * @param fs           fifo segment
- */
-void fifo_segment_update_free_bytes (fifo_segment_t * fs);
-
 /**
  * Fifo segment number of cached bytes
  *

diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h
index 234fd9f..9182a2a 100644 (file)
--- a/src/svm/fifo_types.h
+++ b/src/svm/fifo_types.h
@@ -107,14 +107,14 @@ typedef struct fifo_segment_slice_
   u32 *num_chunks;                     /**< Allocated chunks by chunk size */
   uword n_fl_chunk_bytes;              /**< Chunk bytes on freelist */
   uword virtual_mem;                   /**< Slice sum of all fifo sizes */
-  clib_spinlock_t chunk_lock;
+  u32 n_chunk_lens;
+  CLIB_CACHE_LINE_ALIGN_MARK (lock);
+  u32 chunk_lock;
 } fifo_segment_slice_t;
 
 struct fifo_segment_header_
 {
   fifo_segment_slice_t *slices;                /** Fixed array of slices */
-  ssvm_shared_header_t *ssvm_sh;       /**< Pointer to fs ssvm shared hdr */
-  uword n_free_bytes;                  /**< Segment free bytes */
   uword n_cached_bytes;                        /**< Cached bytes */
   u32 n_active_fifos;                  /**< Number of active fifos */
   u32 n_reserved_bytes;                        /**< Bytes not to be allocated */
@@ -124,6 +124,9 @@ struct fifo_segment_header_
   u8 high_watermark;                   /**< Memory pressure watermark high */
   u8 low_watermark;                    /**< Memory pressure watermark low */
   u8 pct_first_alloc;                  /**< Pct of fifo size to alloc */
+  CLIB_CACHE_LINE_ALIGN_MARK (allocator);
+  uword byte_index;
+  uword max_byte_index;
 };
 
 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,

diff --git a/src/svm/message_queue.c b/src/svm/message_queue.c
index b381173..e586841 100644 (file)
--- a/src/svm/message_queue.c
+++ b/src/svm/message_queue.c
@@ -38,33 +38,16 @@ svm_msg_q_ring_data (svm_msg_q_ring_t * ring, u32 elt_index)
 }
 
 svm_msg_q_t *
-svm_msg_q_alloc (svm_msg_q_cfg_t * cfg)
+svm_msg_q_init (void *base, svm_msg_q_cfg_t *cfg)
 {
-  svm_msg_q_ring_cfg_t *ring_cfg;
-  uword rings_sz = 0, mq_sz;
   svm_msg_q_ring_t *ring;
-  u8 *base, *rings_ptr;
   vec_header_t *vh;
-  u32 vec_sz, q_sz;
   svm_msg_q_t *mq;
+  u8 *rings_ptr;
+  u32 q_sz;
   int i;
 
-  ASSERT (cfg);
-
-  vec_sz = vec_header_bytes (0) + sizeof (svm_msg_q_ring_t) * cfg->n_rings;
-  for (i = 0; i < cfg->n_rings; i++)
-    {
-      if (cfg->ring_cfgs[i].data)
-       continue;
-      ring_cfg = &cfg->ring_cfgs[i];
-      rings_sz += (uword) ring_cfg->nitems * ring_cfg->elsize;
-    }
-
   q_sz = sizeof (svm_queue_t) + cfg->q_nitems * sizeof (svm_msg_q_msg_t);
-  mq_sz = sizeof (svm_msg_q_t) + vec_sz + rings_sz + q_sz;
-  base = clib_mem_alloc_aligned (mq_sz, CLIB_CACHE_LINE_BYTES);
-  if (!base)
-    return 0;
 
   mq = (svm_msg_q_t *) base;
   mq->q = svm_queue_init (base + sizeof (svm_msg_q_t), cfg->q_nitems,
@@ -92,6 +75,60 @@ svm_msg_q_alloc (svm_msg_q_cfg_t * cfg)
   return mq;
 }
 
+uword
+svm_msg_q_size_to_alloc (svm_msg_q_cfg_t *cfg)
+{
+  svm_msg_q_ring_cfg_t *ring_cfg;
+  uword rings_sz = 0, mq_sz;
+  u32 vec_sz, q_sz;
+  int i;
+
+  ASSERT (cfg);
+
+  vec_sz = vec_header_bytes (0) + sizeof (svm_msg_q_ring_t) * cfg->n_rings;
+  for (i = 0; i < cfg->n_rings; i++)
+    {
+      if (cfg->ring_cfgs[i].data)
+       continue;
+      ring_cfg = &cfg->ring_cfgs[i];
+      rings_sz += (uword) ring_cfg->nitems * ring_cfg->elsize;
+    }
+
+  q_sz = sizeof (svm_queue_t) + cfg->q_nitems * sizeof (svm_msg_q_msg_t);
+  mq_sz = sizeof (svm_msg_q_t) + vec_sz + rings_sz + q_sz;
+
+  return mq_sz;
+}
+
+svm_msg_q_t *
+svm_msg_q_alloc (svm_msg_q_cfg_t *cfg)
+{
+  svm_msg_q_ring_cfg_t *ring_cfg;
+  uword rings_sz = 0, mq_sz;
+  u32 vec_sz, q_sz;
+  u8 *base;
+  int i;
+
+  ASSERT (cfg);
+
+  vec_sz = vec_header_bytes (0) + sizeof (svm_msg_q_ring_t) * cfg->n_rings;
+  for (i = 0; i < cfg->n_rings; i++)
+    {
+      if (cfg->ring_cfgs[i].data)
+       continue;
+      ring_cfg = &cfg->ring_cfgs[i];
+      rings_sz += (uword) ring_cfg->nitems * ring_cfg->elsize;
+    }
+
+  q_sz = sizeof (svm_queue_t) + cfg->q_nitems * sizeof (svm_msg_q_msg_t);
+  mq_sz = sizeof (svm_msg_q_t) + vec_sz + rings_sz + q_sz;
+  base = clib_mem_alloc_aligned (mq_sz, CLIB_CACHE_LINE_BYTES);
+  if (!base)
+    return 0;
+
+  return svm_msg_q_init (base, cfg);
+}
+
 void
 svm_msg_q_free (svm_msg_q_t * mq)
 {

diff --git a/src/svm/message_queue.h b/src/svm/message_queue.h
index d4c47d7..50f79fb 100644 (file)
--- a/src/svm/message_queue.h
+++ b/src/svm/message_queue.h
@@ -78,6 +78,8 @@ typedef union
  * @return             message queue
  */
 svm_msg_q_t *svm_msg_q_alloc (svm_msg_q_cfg_t * cfg);
+svm_msg_q_t *svm_msg_q_init (void *base, svm_msg_q_cfg_t *cfg);
+uword svm_msg_q_size_to_alloc (svm_msg_q_cfg_t *cfg);
 
 /**
  * Free message queue

diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index 8441d54..bb3d7ad 100644 (file)
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -874,7 +874,7 @@ segment_manager_alloc_queue (fifo_segment_t * segment,
   u32 fifo_evt_size, session_evt_size = 256, notif_q_size;
   svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
   svm_msg_q_t *q;
-  void *oldheap;
+  void *base;
 
   fifo_evt_size = sizeof (session_event_t);
   notif_q_size = clib_max (16, props->evt_q_size >> 4);
@@ -889,10 +889,8 @@ segment_manager_alloc_queue (fifo_segment_t * segment,
   cfg->q_nitems = props->evt_q_size;
   cfg->ring_cfgs = rc;
 
-  oldheap = ssvm_push_heap (segment->ssvm.sh);
-  q = svm_msg_q_alloc (cfg);
-  fifo_segment_update_free_bytes (segment);
-  ssvm_pop_heap (oldheap);
+  base = fifo_segment_alloc (segment, svm_msg_q_size_to_alloc (cfg));
+  q = svm_msg_q_init (base, cfg);
 
   if (props->use_mq_eventfd)
     {

diff --git a/src/vppinfra/atomics.h b/src/vppinfra/atomics.h
index d6616a9..170b54a 100644 (file)
--- a/src/vppinfra/atomics.h
+++ b/src/vppinfra/atomics.h
@@ -38,6 +38,9 @@
 #define clib_atomic_bool_cmp_and_swap(addr,old,new) __sync_bool_compare_and_swap(addr, old, new)
 
 #define clib_atomic_cmp_and_swap_acq_relax_n(addr,exp,new,weak) __atomic_compare_exchange_n ((addr), (exp), (new), (weak), __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)
+#define clib_atomic_cmp_and_swap_acq_relax(addr, exp, new, weak)              \
+  __atomic_compare_exchange ((addr), (exp), (new), (weak), __ATOMIC_ACQUIRE,  \
+                            __ATOMIC_RELAXED)
 
 #define clib_atomic_test_and_set(a) __atomic_exchange_n(a, 1, __ATOMIC_ACQUIRE)
 #define clib_atomic_release(a) __atomic_store_n(a, 0, __ATOMIC_RELEASE)