X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Ffifo_segment.c;h=49cd1161a4bfeb5d2dda75a6a4030b58c2e14da7;hb=f22f4e562e1b922cff036ef628b77fd2d479d015;hp=01fa65623c52b96ca4db4890cda21f4c456ac50b;hpb=ef4f3e7fea359f651c548182a7597abc066ca372;p=vpp.git

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 01fa65623c5..49cd1161a4b 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -26,11 +26,11 @@
  * @return		number of free bytes
  */
 static uword
-fs_free_space (fifo_segment_t * fs)
+fsh_free_space (fifo_segment_header_t * fsh)
 {
   struct dlmallinfo dlminfo;
 
-  dlminfo = mspace_mallinfo (fs->ssvm.sh->heap);
+  dlminfo = mspace_mallinfo (fsh->ssvm_sh->heap);
   return dlminfo.fordblks;
 }
 
@@ -43,7 +43,31 @@ fsh_free_bytes_sub (fifo_segment_header_t * fsh, int size)
 static inline uword
 fsh_n_free_bytes (fifo_segment_header_t * fsh)
 {
-  return clib_atomic_load_relax_n (&fsh->n_free_bytes);
+  uword n_free = clib_atomic_load_relax_n (&fsh->n_free_bytes);
+  return n_free > fsh->n_reserved_bytes ? n_free - fsh->n_reserved_bytes : 0;
+}
+
+static inline void
+fsh_update_free_bytes (fifo_segment_header_t * fsh)
+{
+  clib_atomic_store_rel_n (&fsh->n_free_bytes, fsh_free_space (fsh));
+}
+
+static void
+fsh_check_mem (fifo_segment_header_t * fsh)
+{
+  uword thresh;
+
+  if (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT)
+    return;
+
+  thresh = clib_max (0.01 * fsh->ssvm_sh->ssvm_size,
+		     2 * fsh->n_reserved_bytes);
+  if (fsh->n_free_bytes > thresh)
+    return;
+
+  fsh->flags |= FIFO_SEGMENT_F_MEM_LIMIT;
+  fsh_update_free_bytes (fsh);
 }
 
 static inline fifo_segment_slice_t *
@@ -67,7 +91,7 @@ fifo_segment_init (fifo_segment_t * fs)
   fifo_segment_header_t *fsh;
   fifo_segment_slice_t *fss;
   ssvm_shared_header_t *sh;
-  u32 max_chunk_sz;
+  u32 max_chunk_sz, max_chunks;
   uword max_fifo;
   void *oldheap;
   int i;
@@ -91,7 +115,7 @@ fifo_segment_init (fifo_segment_t * fs)
 
   fsh->ssvm_sh = fs->ssvm.sh;
   fsh->n_slices = fs->n_slices;
-  max_fifo = clib_min ((fs_free_space (fs) - 4096) / 2,
+  max_fifo = clib_min ((fsh_free_space (fsh) - 4096) / 2,
 		       FIFO_SEGMENT_MAX_FIFO_SIZE);
   fsh->max_log2_chunk_size = max_log2 (max_fifo);
 
@@ -103,11 +127,14 @@ fifo_segment_init (fifo_segment_t * fs)
     {
       fss = fsh_slice_get (fsh, i);
       vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0);
+      clib_spinlock_init (&fss->chunk_lock);
     }
 
   ssvm_pop_heap (oldheap);
 
-  fsh->n_free_bytes = fs_free_space (fs);
+  fsh->n_free_bytes = fsh_free_space (fsh);
+  max_chunks = fsh->n_free_bytes / FIFO_SEGMENT_MIN_FIFO_SIZE;
+  fsh->n_reserved_bytes = (max_chunks / 4) * sizeof (rb_node_t);
   sh->ready = 1;
   return (0);
 }
@@ -217,6 +244,8 @@ fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva,
 static inline u32
 fs_freelist_for_size (u32 size)
 {
+  if (PREDICT_FALSE (size < FIFO_SEGMENT_MIN_FIFO_SIZE))
+    return 0;
   return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE;
 }
 
@@ -252,9 +281,8 @@ fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss,
 
   fss->free_fifos = f->next;
   fss->free_chunks[fl_index] = c->next;
-  c->next = c;
+  c->next = 0;
   c->start_byte = 0;
-  c->length = data_bytes;
   memset (f, 0, sizeof (*f));
   f->start_chunk = c;
   f->end_chunk = c;
@@ -288,7 +316,10 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh,
       fss->free_fifos = f->next;
     }
 
-  fl_index = fs_freelist_for_size (data_bytes) - 1;
+  fl_index = fs_freelist_for_size (data_bytes);
+  if (fl_index > 0)
+    fl_index -= 1;
+
   fl_size = fs_freelist_index_to_size (fl_index);
 
   while (data_bytes)
@@ -302,19 +333,44 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh,
 	  c->next = first;
 	  first = c;
 	  n_alloc += fl_size;
-	  c->length = clib_min (fl_size, data_bytes);
 	  data_bytes -= c->length;
 	}
       else
 	{
-	  ASSERT (fl_index > 0);
+	  /* Failed to allocate with smaller chunks */
+	  if (fl_index == 0)
+	    {
+	      /* free all chunks if any allocated */
+	      c = first;
+	      while (c)
+		{
+		  fl_index = fs_freelist_for_size (c->length);
+		  fl_size = fs_freelist_index_to_size (fl_index);
+		  c->next = fss->free_chunks[fl_index];
+		  fss->free_chunks[fl_index] = c;
+		  fss->n_fl_chunk_bytes += fl_size;
+		  data_bytes += fl_size;
+		}
+	      first = last = 0;
+	      fl_index = fs_freelist_for_size (data_bytes);
+	      if (fss->free_chunks[fl_index + 1])
+		{
+		  fl_index += 1;
+		  fl_size = fs_freelist_index_to_size (fl_index);
+		  continue;
+		}
+
+	      f->next = fss->free_fifos;
+	      fss->free_fifos = f;
+	      return 0;
+	    }
 	  fl_index -= 1;
 	  fl_size = fl_size >> 1;
 	}
     }
+
   f->start_chunk = first;
   f->end_chunk = last;
-  last->next = first;
   fss->n_fl_chunk_bytes -= n_alloc;
   return f;
 }
@@ -324,16 +380,17 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
 			 fifo_segment_slice_t * fss,
 			 u32 fl_index, u32 batch_size)
 {
-  u32 size, hdrs, rounded_data_size;
+  u32 hdrs, rounded_data_size;
   svm_fifo_chunk_t *c;
   svm_fifo_t *f;
   void *oldheap;
+  uword size;
   u8 *fmem;
   int i;
 
   rounded_data_size = fs_freelist_index_to_size (fl_index);
   hdrs = sizeof (*f) + sizeof (*c);
-  size = (hdrs + rounded_data_size) * batch_size;
+  size = (uword) (hdrs + rounded_data_size) * batch_size;
 
   oldheap = ssvm_push_heap (fsh->ssvm_sh);
   fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES,
@@ -355,6 +412,8 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
       c = (svm_fifo_chunk_t *) (fmem + sizeof (*f));
       c->start_byte = 0;
       c->length = rounded_data_size;
+      c->enq_rb_index = RBTREE_TNIL_INDEX;
+      c->deq_rb_index = RBTREE_TNIL_INDEX;
       c->next = fss->free_chunks[fl_index];
       fss->free_chunks[fl_index] = c;
       fmem += hdrs + rounded_data_size;
@@ -393,6 +452,8 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
       if (f)
 	goto done;
     }
+
+  fsh_check_mem (fsh);
   n_free_bytes = fsh_n_free_bytes (fsh);
   if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < n_free_bytes)
     {
@@ -406,7 +467,7 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
   if (fifo_sz <= n_free_bytes)
     {
       void *oldheap = ssvm_push_heap (fsh->ssvm_sh);
-      f = svm_fifo_create (data_bytes);
+      f = svm_fifo_alloc (data_bytes);
       ssvm_pop_heap (oldheap);
       if (f)
 	{
@@ -419,9 +480,87 @@ fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
 
 done:
 
+  if (f)
+    f->fs_hdr = fsh;
   return f;
 }
 
+svm_fifo_chunk_t *
+fsh_alloc_chunk (fifo_segment_header_t * fsh, u32 slice_index, u32 chunk_size)
+{
+  fifo_segment_slice_t *fss;
+  svm_fifo_chunk_t *c;
+  void *oldheap;
+  int fl_index;
+
+  fl_index = fs_freelist_for_size (chunk_size);
+  fss = fsh_slice_get (fsh, slice_index);
+
+  clib_spinlock_lock (&fss->chunk_lock);
+  c = fss->free_chunks[fl_index];
+
+  if (!c)
+    {
+      fsh_check_mem (fsh);
+      chunk_size = fs_freelist_index_to_size (fl_index);
+      if (fsh_n_free_bytes (fsh) < chunk_size)
+	goto done;
+
+      oldheap = ssvm_push_heap (fsh->ssvm_sh);
+      c = svm_fifo_chunk_alloc (chunk_size);
+      ssvm_pop_heap (oldheap);
+
+      if (!c)
+	goto done;
+
+      fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c));
+    }
+  else
+    {
+      fss->free_chunks[fl_index] = c->next;
+      c->next = 0;
+      fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
+    }
+
+done:
+
+  clib_spinlock_unlock (&fss->chunk_lock);
+
+  return c;
+}
+
+static void
+fsh_slice_collect_chunks (fifo_segment_slice_t * fss, svm_fifo_chunk_t * cur)
+{
+  svm_fifo_chunk_t *next;
+  int fl_index;
+
+  clib_spinlock_lock (&fss->chunk_lock);
+
+  while (cur)
+    {
+      next = cur->next;
+      fl_index = fs_freelist_for_size (cur->length);
+      cur->next = fss->free_chunks[fl_index];
+      cur->enq_rb_index = RBTREE_TNIL_INDEX;
+      cur->deq_rb_index = RBTREE_TNIL_INDEX;
+      fss->free_chunks[fl_index] = cur;
+      fss->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index);
+      cur = next;
+    }
+
+  clib_spinlock_unlock (&fss->chunk_lock);
+}
+
+void
+fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index,
+		    svm_fifo_chunk_t * cur)
+{
+  fifo_segment_slice_t *fss;
+  fss = fsh_slice_get (fsh, slice_index);
+  fsh_slice_collect_chunks (fss, cur);
+}
+
 /**
  * Allocate fifo in fifo segment
  */
@@ -442,17 +581,8 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
 
   f->slice_index = slice_index;
 
-  /* (re)initialize the fifo, as in svm_fifo_create */
   svm_fifo_init (f, data_bytes);
 
-  /* Initialize chunks and rbtree for multi-chunk fifos */
-  if (f->start_chunk->next != f->start_chunk)
-    {
-      void *oldheap = ssvm_push_heap (fsh->ssvm_sh);
-      svm_fifo_init_chunks (f);
-      ssvm_pop_heap (oldheap);
-    }
-
   /* If rx fifo type add to active fifos list. When cleaning up segment,
    * we need a list of active sessions that should be disconnected. Since
    * both rx and tx fifos keep pointers to the session, it's enough to track
@@ -466,7 +596,14 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
 	}
       fss->fifos = f;
       f->flags |= SVM_FIFO_F_LL_TRACKED;
+
+      svm_fifo_init_ooo_lookup (f, 0 /* ooo enq */ );
     }
+  else
+    {
+      svm_fifo_init_ooo_lookup (f, 1 /* ooo deq */ );
+    }
+
   fsh_active_fifos_update (fsh, 1);
 
 done:
@@ -480,10 +617,7 @@ void
 fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
 {
   fifo_segment_header_t *fsh = fs->h;
-  svm_fifo_chunk_t *cur, *next;
   fifo_segment_slice_t *fss;
-  void *oldheap;
-  int fl_index;
 
   ASSERT (f->refcnt > 0);
 
@@ -510,27 +644,13 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
   fss->free_fifos = f;
 
   /* Free fifo chunks */
-  cur = f->start_chunk;
-  do
-    {
-      next = cur->next;
-      fl_index = fs_freelist_for_size (cur->length);
-      ASSERT (fl_index < vec_len (fss->free_chunks));
-      cur->next = fss->free_chunks[fl_index];
-      fss->free_chunks[fl_index] = cur;
-      fss->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index);
-      cur = next;
-    }
-  while (cur != f->start_chunk);
+  fsh_slice_collect_chunks (fss, f->start_chunk);
 
-  f->start_chunk = f->end_chunk = f->new_chunks = 0;
+  f->start_chunk = f->end_chunk = 0;
   f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
 
-  oldheap = ssvm_push_heap (fsh->ssvm_sh);
-  svm_fifo_free_chunk_lookup (f);
-  ssvm_pop_heap (oldheap);
-
   /* not allocated on segment heap */
+  svm_fifo_free_chunk_lookup (f);
   svm_fifo_free_ooo_data (f);
 
   if (CLIB_DEBUG)
@@ -550,12 +670,12 @@ fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index,
   fifo_segment_slice_t *fss;
   svm_fifo_t *f;
   void *oldheap;
-  u32 size;
+  uword size;
   u8 *fmem;
   int i;
 
   fss = fsh_slice_get (fsh, slice_index);
-  size = (sizeof (*f)) * batch_size;
+  size = (uword) (sizeof (*f)) * batch_size;
 
   oldheap = ssvm_push_heap (fsh->ssvm_sh);
   fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES,
@@ -586,11 +706,12 @@ int
 fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index,
 				   u32 chunk_size, u32 batch_size)
 {
-  u32 size, rounded_data_size, fl_index;
   fifo_segment_header_t *fsh = fs->h;
+  u32 rounded_data_size, fl_index;
   fifo_segment_slice_t *fss;
   svm_fifo_chunk_t *c;
   void *oldheap;
+  uword size;
   u8 *cmem;
   int i;
 
@@ -602,7 +723,7 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index,
 
   fl_index = fs_freelist_for_size (chunk_size);
   rounded_data_size = fs_freelist_index_to_size (fl_index);
-  size = (sizeof (*c) + rounded_data_size) * batch_size;
+  size = (uword) (sizeof (*c) + rounded_data_size) * batch_size;
 
   oldheap = ssvm_push_heap (fsh->ssvm_sh);
   cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES,
@@ -642,10 +763,10 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
 				     u32 * n_fifo_pairs)
 {
   u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
+  u32 hdrs, pairs_per_slice, alloc_now;
   fifo_segment_header_t *fsh = fs->h;
   int rx_fl_index, tx_fl_index, i;
   fifo_segment_slice_t *fss;
-  u32 hdrs, pairs_per_slice;
   uword space_available;
 
   /* Parameter check */
@@ -673,10 +794,11 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
 
   /* Calculate space requirements */
   pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
-  space_available = fs_free_space (fs);
+  space_available = fsh_free_space (fsh);
   pairs_to_alloc = space_available / pair_size;
   pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs);
   pairs_per_slice = pairs_to_alloc / fs->n_slices;
+  pairs_per_slice += pairs_to_alloc % fs->n_slices ? 1 : 0;
 
   if (!pairs_per_slice)
     return;
@@ -684,81 +806,15 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
   for (i = 0; i < fs->n_slices; i++)
     {
       fss = fsh_slice_get (fsh, i);
-      if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, pairs_to_alloc))
-	clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc);
-      if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, pairs_to_alloc))
-	clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc);
+      alloc_now = clib_min (pairs_per_slice, *n_fifo_pairs);
+      if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, alloc_now))
+	clib_warning ("rx prealloc failed: pairs %u", alloc_now);
+      if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, alloc_now))
+	clib_warning ("tx prealloc failed: pairs %u", alloc_now);
+
+      /* Account for the pairs allocated */
+      *n_fifo_pairs -= alloc_now;
     }
-
-  /* Account for the pairs allocated */
-  *n_fifo_pairs -= pairs_per_slice * fs->n_slices;
-}
-
-int
-fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size)
-{
-  fifo_segment_header_t *fsh = fs->h;
-  fifo_segment_slice_t *fss;
-  svm_fifo_chunk_t *c;
-  void *oldheap;
-  int fl_index;
-
-  fl_index = fs_freelist_for_size (chunk_size);
-  fss = fsh_slice_get (fsh, f->slice_index);
-
-  oldheap = ssvm_push_heap (fsh->ssvm_sh);
-
-  c = fss->free_chunks[fl_index];
-
-  if (!c)
-    {
-      c = svm_fifo_chunk_alloc (chunk_size);
-      if (!c)
-	{
-	  ssvm_pop_heap (oldheap);
-	  return -1;
-	}
-      fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c));
-    }
-  else
-    {
-      fss->free_chunks[fl_index] = c->next;
-      c->next = 0;
-      fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
-    }
-
-  svm_fifo_add_chunk (f, c);
-
-  ssvm_pop_heap (oldheap);
-  return 0;
-}
-
-int
-fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f)
-{
-  fifo_segment_header_t *fsh = fs->h;
-  svm_fifo_chunk_t *cur, *next;
-  fifo_segment_slice_t *fss;
-  void *oldheap;
-  int fl_index;
-
-  oldheap = ssvm_push_heap (fsh->ssvm_sh);
-  cur = svm_fifo_collect_chunks (f);
-
-  fss = fsh_slice_get (fsh, f->slice_index);
-
-  while (cur)
-    {
-      next = cur->next;
-      fl_index = fs_freelist_for_size (cur->length);
-      cur->next = fss->free_chunks[fl_index];
-      fss->free_chunks[fl_index] = cur;
-      cur = next;
-    }
-
-  ssvm_pop_heap (oldheap);
-
-  return 0;
 }
 
 /**
@@ -866,8 +922,7 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
 void
 fifo_segment_update_free_bytes (fifo_segment_t * fs)
 {
-  fifo_segment_header_t *fsh = fs->h;
-  clib_atomic_store_rel_n (&fsh->n_free_bytes, fs_free_space (fs));
+  fsh_update_free_bytes (fs->h);
 }
 
 uword
@@ -957,12 +1012,6 @@ format_fifo_segment (u8 * s, va_list * args)
   int i;
 
   indent = format_get_indent (s) + 2;
-#if USE_DLMALLOC == 0
-  s = format (s, "%U segment heap: %U\n", format_white_space, indent,
-	      format_mheap, fsh->ssvm_sh->heap, verbose);
-  s = format (s, "%U segment has %u active fifos\n",
-	      format_white_space, indent, fifo_segment_num_fifos (fsh));
-#endif
 
   if (fs == 0)
     {
@@ -1028,8 +1077,10 @@ format_fifo_segment (u8 * s, va_list * args)
 	      format_white_space, indent + 2, format_memory_size, chunk_bytes,
 	      chunk_bytes, format_memory_size, est_chunk_bytes,
 	      est_chunk_bytes);
-  s = format (s, "%Ufifo hdr free bytes: %U (%u)\n", format_white_space,
-	      indent + 2, format_memory_size, fifo_hdr, fifo_hdr);
+  s = format (s, "%Ufifo hdr free bytes: %U (%u) reserved %U (%lu)\n",
+	      format_white_space, indent + 2, format_memory_size, fifo_hdr,
+	      fifo_hdr, format_memory_size, fsh->n_reserved_bytes,
+	      fsh->n_reserved_bytes);
   s = format (s, "\n");
 
   return s;