X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer_funcs.h;h=7480326ee3db0c4a76fa4765a0ba98f76bafc6ce;hb=859b59133c143b09fed86af1d409d320fa45424f;hp=f2ac0bd92dd5ebfd79bfec993e51ed08939b2b0b;hpb=3e94c419e3398e0c030ed0d3d3c72cac38397fe6;p=vpp.git

diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index f2ac0bd92dd..7480326ee3d 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -712,7 +712,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
   u8 buffer_pool_index = ~0;
   u32 n_queue = 0, queue[queue_size + 4];
   vlib_buffer_t bt = { };
-#if defined(CLIB_HAVE_VEC128) && !__aarch64__
+#if defined(CLIB_HAVE_VEC128)
   vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
   vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
   vlib_buffer_t flags_refs_mask = {
@@ -737,7 +737,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
       vlib_prefetch_buffer_header (b[6], LOAD);
       vlib_prefetch_buffer_header (b[7], LOAD);
 
-#if defined(CLIB_HAVE_VEC128) && !__aarch64__
+#if defined(CLIB_HAVE_VEC128)
       u8x16 p0, p1, p2, p3, r;
       p0 = u8x16_load_unaligned (b[0]);
       p1 = u8x16_load_unaligned (b[1]);
@@ -815,7 +815,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 	    }
 
 	  buffer_pool_index = b[0]->buffer_pool_index;
-#if defined(CLIB_HAVE_VEC128) && !__aarch64__
+#if defined(CLIB_HAVE_VEC128)
 	  bpi_vec.buffer_pool_index = buffer_pool_index;
 #endif
 	  bp = vlib_get_buffer_pool (vm, buffer_pool_index);
@@ -838,7 +838,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 	  n_queue = 0;
 	}
 
-      if (flags & VLIB_BUFFER_NEXT_PRESENT)
+      if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
 	{
 	  bi = next;
 	  goto next_in_chain;
@@ -954,13 +954,18 @@ vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
 			  u32 n_data_bytes);
 
+/* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
+#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK                     	\
+  (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID |	\
+   VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
+
 /* duplicate all buffers in chain */
 always_inline vlib_buffer_t *
 vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
 {
   vlib_buffer_t *s, *d, *fd;
   uword n_alloc, n_buffers = 1;
-  u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
+  u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK;
   int i;
 
   s = b;
@@ -987,6 +992,7 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
   d->current_data = s->current_data;
   d->current_length = s->current_length;
   d->flags = s->flags & flag_mask;
+  d->trace_handle = s->trace_handle;
   d->total_length_not_including_first_buffer =
     s->total_length_not_including_first_buffer;
   clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
@@ -1033,6 +1039,31 @@ vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
   return d;
 }
 
+/*  \brief Move packet from current position to offset position in buffer.
+    Only work for small packet using one buffer with room to fit the move
+    @param vm - (vlib_main_t *) vlib main data structure pointer
+    @param b -  (vlib_buffer_t *) pointer to buffer
+    @param offset - (i16) position to move the packet in buffer
+ */
+always_inline void
+vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset)
+{
+  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
+  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
+  ASSERT (offset + b->current_length <
+	  vlib_buffer_get_default_data_size (vm));
+
+  u8 *source = vlib_buffer_get_current (b);
+  b->current_data = offset;
+  u8 *destination = vlib_buffer_get_current (b);
+  u16 length = b->current_length;
+
+  if (source + length <= destination)	/* no overlap */
+    clib_memcpy_fast (destination, source, length);
+  else
+    memmove (destination, source, length);
+}
+
 /** \brief Create a maximum of 256 clones of buffer and store them
     in the supplied array
 
@@ -1042,12 +1073,14 @@ vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
     @param n_buffers - (u16) number of buffer clones requested (<=256)
     @param head_end_offset - (u16) offset relative to current position
            where packet head ends
+    @param offset - (i16) copy packet head at current position if 0,
+           else at offset position to change headroom space as specified
     @return - (u16) number of buffers actually cloned, may be
     less than the number requested or zero
 */
 always_inline u16
 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
-		       u16 n_buffers, u16 head_end_offset)
+		       u16 n_buffers, u16 head_end_offset, i16 offset)
 {
   u16 i;
   vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
@@ -1055,10 +1088,16 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
   ASSERT (s->ref_count == 1);
   ASSERT (n_buffers);
   ASSERT (n_buffers <= 256);
+  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
+  ASSERT ((offset + head_end_offset) <
+	  vlib_buffer_get_default_data_size (vm));
 
   if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
     {
       buffers[0] = src_buffer;
+      if (offset)
+	vlib_buffer_move (vm, s, offset);
+
       for (i = 1; i < n_buffers; i++)
 	{
 	  vlib_buffer_t *d;
@@ -1071,7 +1110,7 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
       return n_buffers;
     }
 
-  if (PREDICT_FALSE (n_buffers == 1))
+  if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
     {
       buffers[0] = src_buffer;
       return 1;
@@ -1083,7 +1122,11 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
   for (i = 0; i < n_buffers; i++)
     {
       vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
-      d->current_data = s->current_data;
+      if (offset)
+	d->current_data = offset;
+      else
+	d->current_data = s->current_data;
+
       d->current_length = head_end_offset;
       ASSERT (d->buffer_pool_index == s->buffer_pool_index);
 
@@ -1094,8 +1137,9 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
 	  d->total_length_not_including_first_buffer +=
 	    s->total_length_not_including_first_buffer;
 	}
-      d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
-      d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
+      d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) |
+	VLIB_BUFFER_NEXT_PRESENT;
+      d->trace_handle = s->trace_handle;
       clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
       clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
       clib_memcpy_fast (vlib_buffer_get_current (d),
@@ -1122,12 +1166,14 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
     @param n_buffers - (u16) number of buffer clones requested (<=256)
     @param head_end_offset - (u16) offset relative to current position
            where packet head ends
+    @param offset - (i16) copy packet head at current position if 0,
+           else at offset position to change headroom space as specified
     @return - (u16) number of buffers actually cloned, may be
     less than the number requested or zero
 */
 always_inline u16
-vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
-		   u16 n_buffers, u16 head_end_offset)
+vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+			     u16 n_buffers, u16 head_end_offset, i16 offset)
 {
   vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
   u16 n_cloned = 0;
@@ -1139,16 +1185,36 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
       n_cloned += vlib_buffer_clone_256 (vm,
 					 vlib_get_buffer_index (vm, copy),
 					 (buffers + n_cloned),
-					 256, head_end_offset);
+					 256, head_end_offset, offset);
       n_buffers -= 256;
     }
   n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
 				     buffers + n_cloned,
-				     n_buffers, head_end_offset);
+				     n_buffers, head_end_offset, offset);
 
   return n_cloned;
 }
 
+/** \brief Create multiple clones of buffer and store them
+    in the supplied array
+
+    @param vm - (vlib_main_t *) vlib main data structure pointer
+    @param src_buffer - (u32) source buffer index
+    @param buffers - (u32 * ) buffer index array
+    @param n_buffers - (u16) number of buffer clones requested (<=256)
+    @param head_end_offset - (u16) offset relative to current position
+           where packet head ends
+    @return - (u16) number of buffers actually cloned, may be
+    less than the number requested or zero
+*/
+always_inline u16
+vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
+		   u16 n_buffers, u16 head_end_offset)
+{
+  return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
+				      head_end_offset, 0);
+}
+
 /** \brief Attach cloned tail to the buffer
 
     @param vm - (vlib_main_t *) vlib main data structure pointer
@@ -1357,13 +1423,19 @@ vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
 
       if (dst_left == 0)
 	{
-	  if (db != first)
-	    db->current_data = 0;
 	  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
 	  ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
 	  db = vlib_get_buffer (vm, db->next_buffer);
 	  dst_left = data_size;
-	  dp = db->data;
+	  if (db->current_data > 0)
+	    {
+	      db->current_data = 0;
+	    }
+	  else
+	    {
+	      dst_left += -db->current_data;
+	    }
+	  dp = vlib_buffer_get_current (db);
 	}
 
       while (src_left == 0)