2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_funcs.h: VLIB buffer related functions/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
51 vlib buffer access methods.
55 vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
57 vlib_buffer_main_t *bm = vm->buffer_main;
58 vlib_buffer_pool_t *bp;
60 /* reference count in allocated buffer always must be 1 or higher */
61 ASSERT (b->ref_count > 0);
63 /* verify that buffer pool index is valid */
64 bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index);
65 ASSERT (pointer_to_uword (b) >= bp->start);
66 ASSERT (pointer_to_uword (b) < bp->start + bp->size -
67 (bp->data_size + sizeof (vlib_buffer_t)));
71 vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
74 offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
75 return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
78 /** \brief Translate buffer index into buffer pointer
80 @param vm - (vlib_main_t *) vlib main data structure pointer
81 @param buffer_index - (u32) buffer index
82 @return - (vlib_buffer_t *) buffer pointer
84 always_inline vlib_buffer_t *
85 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
87 vlib_buffer_main_t *bm = vm->buffer_main;
90 b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
91 vlib_buffer_validate (vm, b);
95 static_always_inline u32
96 vlib_buffer_get_default_data_size (vlib_main_t * vm)
98 return vm->buffer_main->default_data_size;
101 static_always_inline void
102 vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
104 #if defined(CLIB_HAVE_VEC512)
105 while (n_indices >= 16)
107 u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
114 #if defined(CLIB_HAVE_VEC256)
115 while (n_indices >= 8)
117 u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
124 #if defined(CLIB_HAVE_VEC128)
125 while (n_indices >= 4)
127 u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
144 vlib_buffer_copy_indices_from_ring (u32 * dst, u32 * ring, u32 start,
145 u32 ring_size, u32 n_buffers)
147 ASSERT (n_buffers <= ring_size);
149 if (PREDICT_TRUE (start + n_buffers <= ring_size))
151 vlib_buffer_copy_indices (dst, ring + start, n_buffers);
155 u32 n = ring_size - start;
156 vlib_buffer_copy_indices (dst, ring + start, n);
157 vlib_buffer_copy_indices (dst + n, ring, n_buffers - n);
162 vlib_buffer_copy_indices_to_ring (u32 * ring, u32 * src, u32 start,
163 u32 ring_size, u32 n_buffers)
165 ASSERT (n_buffers <= ring_size);
167 if (PREDICT_TRUE (start + n_buffers <= ring_size))
169 vlib_buffer_copy_indices (ring + start, src, n_buffers);
173 u32 n = ring_size - start;
174 vlib_buffer_copy_indices (ring + start, src, n);
175 vlib_buffer_copy_indices (ring, src + n, n_buffers - n);
179 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
180 static_always_inline void
181 vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
183 #if defined CLIB_HAVE_VEC512
184 b->as_u8x64[0] = bt->as_u8x64[0];
185 #elif defined (CLIB_HAVE_VEC256)
186 b->as_u8x32[0] = bt->as_u8x32[0];
187 b->as_u8x32[1] = bt->as_u8x32[1];
188 #elif defined (CLIB_HAVE_VEC128)
189 b->as_u8x16[0] = bt->as_u8x16[0];
190 b->as_u8x16[1] = bt->as_u8x16[1];
191 b->as_u8x16[2] = bt->as_u8x16[2];
192 b->as_u8x16[3] = bt->as_u8x16[3];
194 clib_memcpy_fast (b, bt, 64);
199 vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node)
201 ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
202 return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
205 /** \brief Translate array of buffer indices into buffer pointers with offset
207 @param vm - (vlib_main_t *) vlib main data structure pointer
208 @param bi - (u32 *) array of buffer indices
209 @param b - (void **) array to store buffer pointers
210 @param count - (uword) number of elements
211 @param offset - (i32) offset applied to each pointer
213 static_always_inline void
214 vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
217 uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
218 #ifdef CLIB_HAVE_VEC256
219 u64x4 off = u64x4_splat (buffer_mem_start + offset);
220 /* if count is not const, compiler will not unroll while loop
221 se we maintain two-in-parallel variant */
224 u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
225 u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
226 /* shift and add to get vlib_buffer_t pointer */
227 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
228 u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
236 #ifdef CLIB_HAVE_VEC256
237 u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
238 /* shift and add to get vlib_buffer_t pointer */
239 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
240 #elif defined (CLIB_HAVE_VEC128)
241 u64x2 off = u64x2_splat (buffer_mem_start + offset);
242 u32x4 bi4 = u32x4_load_unaligned (bi);
243 u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
244 #if defined (__aarch64__)
245 u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4);
247 bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
248 u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
250 u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
251 u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
253 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
254 b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
255 b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
256 b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
264 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
271 /** \brief Translate array of buffer indices into buffer pointers
273 @param vm - (vlib_main_t *) vlib main data structure pointer
274 @param bi - (u32 *) array of buffer indices
275 @param b - (vlib_buffer_t **) array to store buffer pointers
276 @param count - (uword) number of elements
279 static_always_inline void
280 vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
282 vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
285 /** \brief Translate buffer pointer into buffer index
287 @param vm - (vlib_main_t *) vlib main data structure pointer
288 @param p - (void *) buffer pointer
289 @return - (u32) buffer index
293 vlib_get_buffer_index (vlib_main_t * vm, void *p)
295 vlib_buffer_main_t *bm = vm->buffer_main;
296 uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
297 ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
298 ASSERT (offset < bm->buffer_mem_size);
299 ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
300 return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
303 /** \brief Translate array of buffer pointers into buffer indices with offset
305 @param vm - (vlib_main_t *) vlib main data structure pointer
306 @param b - (void **) array of buffer pointers
307 @param bi - (u32 *) array to store buffer indices
308 @param count - (uword) number of elements
309 @param offset - (i32) offset applied to each pointer
311 static_always_inline void
312 vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
313 uword count, i32 offset)
315 #ifdef CLIB_HAVE_VEC256
316 u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
317 u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
321 /* load 4 pointers into 256-bit register */
322 u64x4 v0 = u64x4_load_unaligned (b);
323 u64x4 v1 = u64x4_load_unaligned (b + 4);
329 v0 >>= CLIB_LOG2_CACHE_LINE_BYTES;
330 v1 >>= CLIB_LOG2_CACHE_LINE_BYTES;
332 /* permute 256-bit register so lower u32s of each buffer index are
333 * placed into lower 128-bits */
334 v2 = u32x8_permute ((u32x8) v0, mask);
335 v3 = u32x8_permute ((u32x8) v1, mask);
337 /* extract lower 128-bits and save them to the array of buffer indices */
338 u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
339 u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
347 /* equivalent non-nector implementation */
348 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
349 bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
350 bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
351 bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
358 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
365 /** \brief Translate array of buffer pointers into buffer indices
367 @param vm - (vlib_main_t *) vlib main data structure pointer
368 @param b - (vlib_buffer_t **) array of buffer pointers
369 @param bi - (u32 *) array to store buffer indices
370 @param count - (uword) number of elements
372 static_always_inline void
373 vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi,
376 vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
379 /** \brief Get next buffer in buffer linklist, or zero for end of list.
381 @param vm - (vlib_main_t *) vlib main data structure pointer
382 @param b - (void *) buffer pointer
383 @return - (vlib_buffer_t *) next buffer, or NULL
385 always_inline vlib_buffer_t *
386 vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
388 return (b->flags & VLIB_BUFFER_NEXT_PRESENT
389 ? vlib_get_buffer (vm, b->next_buffer) : 0);
392 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
393 vlib_buffer_t * b_first);
395 /** \brief Get length in bytes of the buffer chain
397 @param vm - (vlib_main_t *) vlib main data structure pointer
398 @param b - (void *) buffer pointer
399 @return - (uword) length of buffer chain
402 vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
404 uword len = b->current_length;
406 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
409 if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
410 return len + b->total_length_not_including_first_buffer;
412 return vlib_buffer_length_in_chain_slow_path (vm, b);
415 /** \brief Get length in bytes of the buffer index buffer chain
417 @param vm - (vlib_main_t *) vlib main data structure pointer
418 @param bi - (u32) buffer index
419 @return - (uword) length of buffer chain
422 vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
424 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
425 return vlib_buffer_length_in_chain (vm, b);
428 /** \brief Copy buffer contents to memory
430 @param vm - (vlib_main_t *) vlib main data structure pointer
431 @param buffer_index - (u32) buffer index
432 @param contents - (u8 *) memory, <strong>must be large enough</strong>
433 @return - (uword) length of buffer chain
436 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
438 uword content_len = 0;
444 b = vlib_get_buffer (vm, buffer_index);
445 l = b->current_length;
446 clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
448 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
450 buffer_index = b->next_buffer;
457 vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
459 return vlib_physmem_get_pa (vm, b->data);
463 vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
465 return vlib_buffer_get_pa (vm, b) + b->current_data;
468 /** \brief Prefetch buffer metadata by buffer index
469 The first 64 bytes of buffer contains most header information
471 @param vm - (vlib_main_t *) vlib main data structure pointer
472 @param bi - (u32) buffer index
473 @param type - LOAD, STORE. In most cases, STORE is the right answer
475 /* Prefetch buffer header given index. */
476 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
478 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
479 vlib_prefetch_buffer_header (_b, type); \
484 /* Index is unknown. */
487 /* Index is known and free/allocated. */
488 VLIB_BUFFER_KNOWN_FREE,
489 VLIB_BUFFER_KNOWN_ALLOCATED,
490 } vlib_buffer_known_state_t;
492 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
494 vlib_buffer_known_state_t
497 always_inline vlib_buffer_known_state_t
498 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
500 vlib_buffer_main_t *bm = vm->buffer_main;
502 clib_spinlock_lock (&bm->buffer_known_hash_lockp);
503 uword *p = hash_get (bm->buffer_known_hash, buffer_index);
504 clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
505 return p ? p[0] : VLIB_BUFFER_UNKNOWN;
508 /* Validates sanity of a single buffer.
509 Returns format'ed vector with error message if any. */
510 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
513 u8 *vlib_validate_buffers (vlib_main_t * vm,
515 uword next_buffer_stride,
517 vlib_buffer_known_state_t known_state,
518 uword follow_buffer_next);
520 static_always_inline vlib_buffer_pool_t *
521 vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
523 vlib_buffer_main_t *bm = vm->buffer_main;
524 return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
527 static_always_inline uword
528 vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
531 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
534 ASSERT (bp->buffers);
536 clib_spinlock_lock (&bp->lock);
538 if (PREDICT_TRUE (n_buffers < len))
541 vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
543 clib_spinlock_unlock (&bp->lock);
548 vlib_buffer_copy_indices (buffers, bp->buffers, len);
550 clib_spinlock_unlock (&bp->lock);
556 /** \brief Allocate buffers from specific pool into supplied array
558 @param vm - (vlib_main_t *) vlib main data structure pointer
559 @param buffers - (u32 * ) buffer index array
560 @param n_buffers - (u32) number of buffers requested
561 @return - (u32) number of buffers actually allocated, may be
562 less than the number requested or zero
566 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
567 u8 buffer_pool_index)
569 vlib_buffer_main_t *bm = vm->buffer_main;
570 vlib_buffer_pool_t *bp;
571 vlib_buffer_pool_thread_t *bpt;
572 u32 *src, *dst, len, n_left;
574 bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
575 bpt = vec_elt_at_index (bp->threads, vm->thread_index);
581 /* per-thread cache contains enough buffers */
582 if (len >= n_buffers)
584 src = bpt->cached_buffers + len - n_buffers;
585 vlib_buffer_copy_indices (dst, src, n_buffers);
586 bpt->n_cached -= n_buffers;
589 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
590 VLIB_BUFFER_KNOWN_FREE);
594 /* alloc bigger than cache - take buffers directly from main pool */
595 if (n_buffers >= VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ)
597 n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers,
601 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
602 VLIB_BUFFER_KNOWN_FREE);
606 /* take everything available in the cache */
609 vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
615 len = round_pow2 (n_left, 32);
616 len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
622 u32 n_copy = clib_min (len, n_left);
623 src = bpt->cached_buffers + len - n_copy;
624 vlib_buffer_copy_indices (dst, src, n_copy);
625 bpt->n_cached -= n_copy;
631 /* Verify that buffers are known free. */
633 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
634 VLIB_BUFFER_KNOWN_FREE);
639 /** \brief Allocate buffers from specific numa node into supplied array
641 @param vm - (vlib_main_t *) vlib main data structure pointer
642 @param buffers - (u32 * ) buffer index array
643 @param n_buffers - (u32) number of buffers requested
644 @param numa_node - (u32) numa node
645 @return - (u32) number of buffers actually allocated, may be
646 less than the number requested or zero
649 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
652 u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
653 return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
656 /** \brief Allocate buffers into supplied array
658 @param vm - (vlib_main_t *) vlib main data structure pointer
659 @param buffers - (u32 * ) buffer index array
660 @param n_buffers - (u32) number of buffers requested
661 @return - (u32) number of buffers actually allocated, may be
662 less than the number requested or zero
666 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
668 return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
671 /** \brief Allocate buffers into ring
673 @param vm - (vlib_main_t *) vlib main data structure pointer
674 @param buffers - (u32 * ) buffer index ring
675 @param start - (u32) first slot in the ring
676 @param ring_size - (u32) ring size
677 @param n_buffers - (u32) number of buffers requested
678 @return - (u32) number of buffers actually allocated, may be
679 less than the number requested or zero
682 vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
683 u32 ring_size, u32 n_buffers)
687 ASSERT (n_buffers <= ring_size);
689 if (PREDICT_TRUE (start + n_buffers <= ring_size))
690 return vlib_buffer_alloc (vm, ring + start, n_buffers);
692 n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
694 if (PREDICT_TRUE (n_alloc == ring_size - start))
695 n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
700 /** \brief Allocate buffers into ring from specific buffer pool
702 @param vm - (vlib_main_t *) vlib main data structure pointer
703 @param buffers - (u32 * ) buffer index ring
704 @param start - (u32) first slot in the ring
705 @param ring_size - (u32) ring size
706 @param n_buffers - (u32) number of buffers requested
707 @return - (u32) number of buffers actually allocated, may be
708 less than the number requested or zero
711 vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start,
712 u32 ring_size, u32 n_buffers,
713 u8 buffer_pool_index)
717 ASSERT (n_buffers <= ring_size);
719 if (PREDICT_TRUE (start + n_buffers <= ring_size))
720 return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
723 n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
726 if (PREDICT_TRUE (n_alloc == ring_size - start))
727 n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
733 static_always_inline void
734 vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
735 u32 * buffers, u32 n_buffers)
737 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
738 vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads,
740 u32 n_cached, n_empty;
743 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
744 VLIB_BUFFER_KNOWN_ALLOCATED);
746 n_cached = bpt->n_cached;
747 n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached;
748 if (n_buffers <= n_empty)
750 vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
752 bpt->n_cached = n_cached + n_buffers;
756 vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
757 buffers + n_buffers - n_empty, n_empty);
758 bpt->n_cached = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ;
760 clib_spinlock_lock (&bp->lock);
761 vlib_buffer_copy_indices (bp->buffers + bp->n_avail, buffers,
762 n_buffers - n_empty);
763 bp->n_avail += n_buffers - n_empty;
764 clib_spinlock_unlock (&bp->lock);
767 static_always_inline void
768 vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
771 const int queue_size = 128;
772 vlib_buffer_pool_t *bp = 0;
773 u8 buffer_pool_index = ~0;
774 u32 n_queue = 0, queue[queue_size + 4];
775 vlib_buffer_t bt = { };
776 #if defined(CLIB_HAVE_VEC128)
777 vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
778 vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
779 vlib_buffer_t flags_refs_mask = {
780 .flags = VLIB_BUFFER_NEXT_PRESENT,
788 u32 bi, sum = 0, flags, next;
793 vlib_get_buffers (vm, buffers, b, 4);
794 vlib_get_buffers (vm, buffers + 8, b + 4, 4);
796 vlib_prefetch_buffer_header (b[4], LOAD);
797 vlib_prefetch_buffer_header (b[5], LOAD);
798 vlib_prefetch_buffer_header (b[6], LOAD);
799 vlib_prefetch_buffer_header (b[7], LOAD);
801 #if defined(CLIB_HAVE_VEC128)
802 u8x16 p0, p1, p2, p3, r;
803 p0 = u8x16_load_unaligned (b[0]);
804 p1 = u8x16_load_unaligned (b[1]);
805 p2 = u8x16_load_unaligned (b[2]);
806 p3 = u8x16_load_unaligned (b[3]);
808 r = p0 ^ bpi_vec.as_u8x16[0];
809 r |= p1 ^ bpi_vec.as_u8x16[0];
810 r |= p2 ^ bpi_vec.as_u8x16[0];
811 r |= p3 ^ bpi_vec.as_u8x16[0];
812 r &= bpi_mask.as_u8x16[0];
813 r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
815 sum = !u8x16_is_all_zero (r);
821 sum &= VLIB_BUFFER_NEXT_PRESENT;
822 sum += b[0]->ref_count - 1;
823 sum += b[1]->ref_count - 1;
824 sum += b[2]->ref_count - 1;
825 sum += b[3]->ref_count - 1;
826 sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
827 sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
828 sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
829 sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
835 vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
836 vlib_buffer_copy_template (b[0], &bt);
837 vlib_buffer_copy_template (b[1], &bt);
838 vlib_buffer_copy_template (b[2], &bt);
839 vlib_buffer_copy_template (b[3], &bt);
842 vlib_buffer_validate (vm, b[0]);
843 vlib_buffer_validate (vm, b[1]);
844 vlib_buffer_validate (vm, b[2]);
845 vlib_buffer_validate (vm, b[3]);
847 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
848 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
849 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
850 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
852 if (n_queue >= queue_size)
854 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
865 b[0] = vlib_get_buffer (vm, bi);
867 next = b[0]->next_buffer;
869 if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
874 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
878 buffer_pool_index = b[0]->buffer_pool_index;
879 #if defined(CLIB_HAVE_VEC128)
880 bpi_vec.buffer_pool_index = buffer_pool_index;
882 bp = vlib_get_buffer_pool (vm, buffer_pool_index);
883 vlib_buffer_copy_template (&bt, &bp->buffer_template);
886 vlib_buffer_validate (vm, b[0]);
888 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
890 if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
892 vlib_buffer_copy_template (b[0], &bt);
893 queue[n_queue++] = bi;
896 if (n_queue == queue_size)
898 vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
902 if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
913 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
917 /** \brief Free buffers
918 Frees the entire buffer chain for each buffer
920 @param vm - (vlib_main_t *) vlib main data structure pointer
921 @param buffers - (u32 * ) buffer index array
922 @param n_buffers - (u32) number of buffers to free
926 vlib_buffer_free (vlib_main_t * vm,
927 /* pointer to first buffer */
929 /* number of buffers to free */
932 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
935 /** \brief Free buffers, does not free the buffer chain for each buffer
937 @param vm - (vlib_main_t *) vlib main data structure pointer
938 @param buffers - (u32 * ) buffer index array
939 @param n_buffers - (u32) number of buffers to free
943 vlib_buffer_free_no_next (vlib_main_t * vm,
944 /* pointer to first buffer */
946 /* number of buffers to free */
949 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
952 /** \brief Free one buffer
953 Shorthand to free a single buffer chain.
955 @param vm - (vlib_main_t *) vlib main data structure pointer
956 @param buffer_index - (u32) buffer index to free
959 vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
961 vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
964 /** \brief Free buffers from ring
966 @param vm - (vlib_main_t *) vlib main data structure pointer
967 @param buffers - (u32 * ) buffer index ring
968 @param start - (u32) first slot in the ring
969 @param ring_size - (u32) ring size
970 @param n_buffers - (u32) number of buffers
973 vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start,
974 u32 ring_size, u32 n_buffers)
976 ASSERT (n_buffers <= ring_size);
978 if (PREDICT_TRUE (start + n_buffers <= ring_size))
980 vlib_buffer_free (vm, ring + start, n_buffers);
984 vlib_buffer_free (vm, ring + start, ring_size - start);
985 vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
989 /** \brief Free buffers from ring without freeing tail buffers
991 @param vm - (vlib_main_t *) vlib main data structure pointer
992 @param buffers - (u32 * ) buffer index ring
993 @param start - (u32) first slot in the ring
994 @param ring_size - (u32) ring size
995 @param n_buffers - (u32) number of buffers
998 vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
999 u32 ring_size, u32 n_buffers)
1001 ASSERT (n_buffers <= ring_size);
1003 if (PREDICT_TRUE (start + n_buffers <= ring_size))
1005 vlib_buffer_free_no_next (vm, ring + start, n_buffers);
1009 vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
1010 vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
1014 /* Append given data to end of buffer, possibly allocating new buffers. */
1015 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
1018 /* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
1019 #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \
1020 (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \
1021 VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
1023 /* duplicate all buffers in chain */
1024 always_inline vlib_buffer_t *
1025 vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
1027 vlib_buffer_t *s, *d, *fd;
1028 uword n_alloc, n_buffers = 1;
1029 u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK;
1033 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1036 s = vlib_get_buffer (vm, s->next_buffer);
1038 u32 new_buffers[n_buffers];
1040 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
1042 /* No guarantee that we'll get all the buffers we asked for */
1043 if (PREDICT_FALSE (n_alloc < n_buffers))
1046 vlib_buffer_free (vm, new_buffers, n_alloc);
1052 fd = d = vlib_get_buffer (vm, new_buffers[0]);
1053 d->current_data = s->current_data;
1054 d->current_length = s->current_length;
1055 d->flags = s->flags & flag_mask;
1056 d->trace_handle = s->trace_handle;
1057 d->total_length_not_including_first_buffer =
1058 s->total_length_not_including_first_buffer;
1059 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1060 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1061 clib_memcpy_fast (vlib_buffer_get_current (d),
1062 vlib_buffer_get_current (s), s->current_length);
1065 for (i = 1; i < n_buffers; i++)
1068 d->next_buffer = new_buffers[i];
1070 s = vlib_get_buffer (vm, s->next_buffer);
1071 d = vlib_get_buffer (vm, new_buffers[i]);
1072 d->current_data = s->current_data;
1073 d->current_length = s->current_length;
1074 clib_memcpy_fast (vlib_buffer_get_current (d),
1075 vlib_buffer_get_current (s), s->current_length);
1076 d->flags = s->flags & flag_mask;
1082 /* duplicate first buffer in chain */
1083 always_inline vlib_buffer_t *
1084 vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
1088 if ((vlib_buffer_alloc (vm, di, 1)) != 1)
1091 d = vlib_get_buffer (vm, *di);
1093 d->current_data = b->current_data;
1094 d->current_length = b->current_length;
1095 clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
1096 clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
1097 clib_memcpy_fast (vlib_buffer_get_current (d),
1098 vlib_buffer_get_current (b), b->current_length);
1103 /* \brief Move packet from current position to offset position in buffer.
1104 Only work for small packet using one buffer with room to fit the move
1105 @param vm - (vlib_main_t *) vlib main data structure pointer
1106 @param b - (vlib_buffer_t *) pointer to buffer
1107 @param offset - (i16) position to move the packet in buffer
1110 vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset)
1112 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1113 ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1114 ASSERT (offset + b->current_length <
1115 vlib_buffer_get_default_data_size (vm));
1117 u8 *source = vlib_buffer_get_current (b);
1118 b->current_data = offset;
1119 u8 *destination = vlib_buffer_get_current (b);
1120 u16 length = b->current_length;
1122 if (source + length <= destination) /* no overlap */
1123 clib_memcpy_fast (destination, source, length);
1125 memmove (destination, source, length);
1128 /** \brief Create a maximum of 256 clones of buffer and store them
1129 in the supplied array
1131 @param vm - (vlib_main_t *) vlib main data structure pointer
1132 @param src_buffer - (u32) source buffer index
1133 @param buffers - (u32 * ) buffer index array
1134 @param n_buffers - (u16) number of buffer clones requested (<=256)
1135 @param head_end_offset - (u16) offset relative to current position
1136 where packet head ends
1137 @param offset - (i16) copy packet head at current position if 0,
1138 else at offset position to change headroom space as specified
1139 @return - (u16) number of buffers actually cloned, may be
1140 less than the number requested or zero
1143 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1144 u16 n_buffers, u16 head_end_offset, i16 offset)
1147 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1149 ASSERT (s->ref_count == 1);
1151 ASSERT (n_buffers <= 256);
1152 ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1153 ASSERT ((offset + head_end_offset) <
1154 vlib_buffer_get_default_data_size (vm));
1156 if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
1158 buffers[0] = src_buffer;
1160 vlib_buffer_move (vm, s, offset);
1162 for (i = 1; i < n_buffers; i++)
1165 d = vlib_buffer_copy (vm, s);
1168 buffers[i] = vlib_get_buffer_index (vm, d);
1174 if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
1176 buffers[0] = src_buffer;
1180 n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
1181 s->buffer_pool_index);
1183 for (i = 0; i < n_buffers; i++)
1185 vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
1187 d->current_data = offset;
1189 d->current_data = s->current_data;
1191 d->current_length = head_end_offset;
1192 ASSERT (d->buffer_pool_index == s->buffer_pool_index);
1194 d->total_length_not_including_first_buffer = s->current_length -
1196 if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
1198 d->total_length_not_including_first_buffer +=
1199 s->total_length_not_including_first_buffer;
1201 d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) |
1202 VLIB_BUFFER_NEXT_PRESENT;
1203 d->trace_handle = s->trace_handle;
1204 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1205 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1206 clib_memcpy_fast (vlib_buffer_get_current (d),
1207 vlib_buffer_get_current (s), head_end_offset);
1208 d->next_buffer = src_buffer;
1210 vlib_buffer_advance (s, head_end_offset);
1211 s->ref_count = n_buffers ? n_buffers : s->ref_count;
1212 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1214 s = vlib_get_buffer (vm, s->next_buffer);
1215 s->ref_count = n_buffers ? n_buffers : s->ref_count;
1221 /** \brief Create multiple clones of buffer and store them
1222 in the supplied array
1224 @param vm - (vlib_main_t *) vlib main data structure pointer
1225 @param src_buffer - (u32) source buffer index
1226 @param buffers - (u32 * ) buffer index array
1227 @param n_buffers - (u16) number of buffer clones requested (<=256)
1228 @param head_end_offset - (u16) offset relative to current position
1229 where packet head ends
1230 @param offset - (i16) copy packet head at current position if 0,
1231 else at offset position to change headroom space as specified
1232 @return - (u16) number of buffers actually cloned, may be
1233 less than the number requested or zero
1236 vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1237 u16 n_buffers, u16 head_end_offset, i16 offset)
1239 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1242 while (n_buffers > 256)
1244 vlib_buffer_t *copy;
1245 copy = vlib_buffer_copy (vm, s);
1246 n_cloned += vlib_buffer_clone_256 (vm,
1247 vlib_get_buffer_index (vm, copy),
1248 (buffers + n_cloned),
1249 256, head_end_offset, offset);
1252 n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
1254 n_buffers, head_end_offset, offset);
1259 /** \brief Create multiple clones of buffer and store them
1260 in the supplied array
1262 @param vm - (vlib_main_t *) vlib main data structure pointer
1263 @param src_buffer - (u32) source buffer index
1264 @param buffers - (u32 * ) buffer index array
1265 @param n_buffers - (u16) number of buffer clones requested (<=256)
1266 @param head_end_offset - (u16) offset relative to current position
1267 where packet head ends
1268 @return - (u16) number of buffers actually cloned, may be
1269 less than the number requested or zero
1272 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1273 u16 n_buffers, u16 head_end_offset)
1275 return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
1276 head_end_offset, 0);
1279 /** \brief Attach cloned tail to the buffer
1281 @param vm - (vlib_main_t *) vlib main data structure pointer
1282 @param head - (vlib_buffer_t *) head buffer
1283 @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
1287 vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
1288 vlib_buffer_t * tail)
1290 ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1291 ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
1293 head->flags |= VLIB_BUFFER_NEXT_PRESENT;
1294 head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1295 head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1296 head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1297 head->next_buffer = vlib_get_buffer_index (vm, tail);
1298 head->total_length_not_including_first_buffer = tail->current_length +
1299 tail->total_length_not_including_first_buffer;
1302 clib_atomic_add_fetch (&tail->ref_count, 1);
1304 if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
1306 tail = vlib_get_buffer (vm, tail->next_buffer);
1311 /* Initializes the buffer as an empty packet with no chained buffers. */
1313 vlib_buffer_chain_init (vlib_buffer_t * first)
1315 first->total_length_not_including_first_buffer = 0;
1316 first->current_length = 0;
1317 first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1318 first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1321 /* The provided next_bi buffer index is appended to the end of the packet. */
1322 always_inline vlib_buffer_t *
1323 vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
1325 vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
1326 last->next_buffer = next_bi;
1327 last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1328 next_buffer->current_length = 0;
1329 next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1333 /* Increases or decreases the packet length.
1334 * It does not allocate or deallocate new buffers.
1335 * Therefore, the added length must be compatible
1336 * with the last buffer. */
1338 vlib_buffer_chain_increase_length (vlib_buffer_t * first,
1339 vlib_buffer_t * last, i32 len)
1341 last->current_length += len;
1343 first->total_length_not_including_first_buffer += len;
1346 /* Copy data to the end of the packet and increases its length.
1347 * It does not allocate new buffers.
1348 * Returns the number of copied bytes. */
1350 vlib_buffer_chain_append_data (vlib_main_t * vm,
1351 vlib_buffer_t * first,
1352 vlib_buffer_t * last, void *data, u16 data_len)
1354 u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
1355 ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
1356 u16 len = clib_min (data_len,
1357 n_buffer_bytes - last->current_length -
1358 last->current_data);
1359 clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
1361 vlib_buffer_chain_increase_length (first, last, len);
1365 /* Copy data to the end of the packet and increases its length.
1366 * Allocates additional buffers from the free list if necessary.
1367 * Returns the number of copied bytes.
1368 * 'last' value is modified whenever new buffers are allocated and
1369 * chained and points to the last buffer in the chain. */
1371 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
1372 vlib_buffer_t * first,
1373 vlib_buffer_t ** last, void *data,
1375 void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);
1377 format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
1378 format_vlib_buffer_contents, format_vlib_buffer_no_chain;
1382 /* Vector of packet data. */
1385 /* Number of buffers to allocate in each call to allocator. */
1386 u32 min_n_buffers_each_alloc;
1389 } vlib_packet_template_t;
1391 void vlib_packet_template_init (vlib_main_t * vm,
1392 vlib_packet_template_t * t,
1394 uword n_packet_data_bytes,
1395 uword min_n_buffers_each_alloc,
1398 void *vlib_packet_template_get_packet (vlib_main_t * vm,
1399 vlib_packet_template_t * t,
1403 vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
1405 vec_free (t->packet_data);
1409 vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b)
1411 return b->data + vlib_buffer_get_default_data_size (vm) -
1412 ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1416 vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
1418 vlib_buffer_t *db = b, *sb, *first = b;
1420 u32 bytes_left = 0, data_size;
1421 u16 src_left, dst_left, n_buffers = 1;
1425 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1428 data_size = vlib_buffer_get_default_data_size (vm);
1430 dst_left = vlib_buffer_space_left_at_end (vm, b);
1432 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1434 b = vlib_get_buffer (vm, b->next_buffer);
1435 if (b->ref_count > 1)
1437 bytes_left += b->current_length;
1441 /* if buffer is cloned, create completely new chain - unless everything fits
1442 * into one buffer */
1443 if (is_cloned && bytes_left >= dst_left)
1446 u32 space_needed = bytes_left - dst_left;
1449 if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1454 b = vlib_get_buffer (vm, tail);
1456 while (len < space_needed)
1459 if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1461 vlib_buffer_free_one (vm, tail);
1464 b->flags = VLIB_BUFFER_NEXT_PRESENT;
1465 b->next_buffer = bi;
1466 b = vlib_get_buffer (vm, bi);
1470 sb = vlib_get_buffer (vm, first->next_buffer);
1471 to_free = first->next_buffer;
1472 first->next_buffer = tail;
1475 sb = vlib_get_buffer (vm, first->next_buffer);
1477 src_left = sb->current_length;
1478 sp = vlib_buffer_get_current (sb);
1479 dp = vlib_buffer_get_tail (db);
1487 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1488 ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1489 db = vlib_get_buffer (vm, db->next_buffer);
1490 dst_left = data_size;
1491 if (db->current_data > 0)
1493 db->current_data = 0;
1497 dst_left += -db->current_data;
1499 dp = vlib_buffer_get_current (db);
1502 while (src_left == 0)
1504 ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1505 sb = vlib_get_buffer (vm, sb->next_buffer);
1506 src_left = sb->current_length;
1507 sp = vlib_buffer_get_current (sb);
1510 bytes_to_copy = clib_min (dst_left, src_left);
1515 bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1517 clib_memcpy_fast (dp, sp, bytes_to_copy);
1520 src_left -= bytes_to_copy;
1521 dst_left -= bytes_to_copy;
1522 dp += bytes_to_copy;
1523 sp += bytes_to_copy;
1524 bytes_left -= bytes_to_copy;
1527 db->current_data = 0;
1528 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1530 if (is_cloned && to_free)
1531 vlib_buffer_free_one (vm, to_free);
1534 if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1535 vlib_buffer_free_one (vm, db->next_buffer);
1536 db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1539 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1541 b = vlib_get_buffer (vm, b->next_buffer);
1546 first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1551 #endif /* included_vlib_buffer_funcs_h */
1554 * fd.io coding-style-patch-verification: ON
1557 * eval: (c-set-style "gnu")