X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=e7a1188d59157956ff7840dc1f15b56fcbd59484;hb=fa76a76;hp=817cb8d2639e1e7d60a0a52e94848103d412c16d;hpb=de9a849a18514f0b09bb5f57a73f6a57ee425c76;p=vpp.git

diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 817cb8d2639..e7a1188d591 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -218,6 +218,7 @@ svm_fifo_create (u32 data_size_in_bytes)
   f->nitems = data_size_in_bytes;
   f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
   f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+  f->segment_index = SVM_FIFO_INVALID_INDEX;
   f->refcnt = 1;
   return (f);
 }
@@ -485,7 +486,7 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
       first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
 	? (nitems - f->tail) : total_copy_bytes;
 
-      _clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes);
+      clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes);
       f->tail += first_copy_bytes;
       f->tail = (f->tail == nitems) ? 0 : f->tail;
 
@@ -493,8 +494,9 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
       second_copy_bytes = total_copy_bytes - first_copy_bytes;
       if (second_copy_bytes)
 	{
-	  _clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes,
-			second_copy_bytes);
+	  clib_memcpy_fast (&f->data[f->tail],
+			    copy_from_here + first_copy_bytes,
+			    second_copy_bytes);
 	  f->tail += second_copy_bytes;
 	  f->tail = (f->tail == nitems) ? 0 : f->tail;
 	}
@@ -518,7 +520,7 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
 
   /* Atomically increase the queue length */
   ASSERT (cursize + total_copy_bytes <= nitems);
-  clib_atomic_fetch_add (&f->cursize, total_copy_bytes);
+  clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes);
 
   return (total_copy_bytes);
 }
@@ -571,8 +573,8 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
   first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
     ? (nitems - normalized_offset) : total_copy_bytes;
 
-  _clib_memcpy (&f->data[normalized_offset], copy_from_here,
-		first_copy_bytes);
+  clib_memcpy_fast (&f->data[normalized_offset], copy_from_here,
+		    first_copy_bytes);
 
   /* Number of bytes in second copy segment, if any */
   second_copy_bytes = total_copy_bytes - first_copy_bytes;
@@ -583,8 +585,8 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
 
       ASSERT (normalized_offset == 0);
 
-      _clib_memcpy (&f->data[normalized_offset],
-		    copy_from_here + first_copy_bytes, second_copy_bytes);
+      clib_memcpy_fast (&f->data[normalized_offset],
+			copy_from_here + first_copy_bytes, second_copy_bytes);
     }
 
   return (0);
@@ -608,11 +610,11 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
   first_chunk = f->nitems - f->head;
   ASSERT (len <= f->nitems);
   if (len <= first_chunk)
-    _clib_memcpy (&f->data[f->head], data, len);
+    clib_memcpy_fast (&f->data[f->head], data, len);
   else
     {
-      _clib_memcpy (&f->data[f->head], data, first_chunk);
-      _clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk);
+      clib_memcpy_fast (&f->data[f->head], data, first_chunk);
+      clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
     }
 }
 #endif
@@ -638,7 +640,7 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
       /* Number of bytes in first copy segment */
       first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
 	? (nitems - f->head) : total_copy_bytes;
-      _clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes);
+      clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes);
       f->head += first_copy_bytes;
       f->head = (f->head == nitems) ? 0 : f->head;
 
@@ -646,8 +648,8 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
       second_copy_bytes = total_copy_bytes - first_copy_bytes;
       if (second_copy_bytes)
 	{
-	  _clib_memcpy (copy_here + first_copy_bytes,
-			&f->data[f->head], second_copy_bytes);
+	  clib_memcpy_fast (copy_here + first_copy_bytes,
+			    &f->data[f->head], second_copy_bytes);
 	  f->head += second_copy_bytes;
 	  f->head = (f->head == nitems) ? 0 : f->head;
 	}
@@ -665,7 +667,7 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
 
   ASSERT (f->head <= nitems);
   ASSERT (cursize >= total_copy_bytes);
-  clib_atomic_fetch_sub (&f->cursize, total_copy_bytes);
+  clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes);
 
   return (total_copy_bytes);
 }
@@ -705,14 +707,14 @@ CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
       first_copy_bytes =
 	((nitems - real_head) < total_copy_bytes) ?
 	(nitems - real_head) : total_copy_bytes;
-      _clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes);
+      clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes);
 
       /* Number of bytes in second copy segment, if any */
       second_copy_bytes = total_copy_bytes - first_copy_bytes;
       if (second_copy_bytes)
 	{
-	  _clib_memcpy (copy_here + first_copy_bytes, &f->data[0],
-			second_copy_bytes);
+	  clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
+			    second_copy_bytes);
 	}
     }
   return total_copy_bytes;
@@ -763,7 +765,7 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
 
   ASSERT (f->head <= nitems);
   ASSERT (cursize >= total_drop_bytes);
-  clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
+  clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
 
   return total_drop_bytes;
 }
@@ -772,7 +774,7 @@ void
 svm_fifo_dequeue_drop_all (svm_fifo_t * f)
 {
   f->head = f->tail;
-  clib_atomic_fetch_sub (&f->cursize, f->cursize);
+  clib_atomic_fetch_sub_rel (&f->cursize, f->cursize);
 }
 
 int
@@ -819,7 +821,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
       f->head = (f->head + fs[0].len) % f->nitems;
       total_drop_bytes = fs[0].len;
     }
-  clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
+  clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
 }
 
 u32