2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_funcs.h: VLIB buffer related functions/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
51 vlib buffer access methods.
55 vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
57 vlib_buffer_main_t *bm = vm->buffer_main;
58 vlib_buffer_pool_t *bp;
60 /* reference count in allocated buffer always must be 1 or higher */
61 ASSERT (b->ref_count > 0);
63 /* verify that buffer pool index is valid */
64 bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index);
65 ASSERT (pointer_to_uword (b) >= bp->start);
66 ASSERT (pointer_to_uword (b) < bp->start + bp->size -
67 (bp->data_size + sizeof (vlib_buffer_t)));
71 vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
74 offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
75 return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
78 /** \brief Translate buffer index into buffer pointer
80 @param vm - (vlib_main_t *) vlib main data structure pointer
81 @param buffer_index - (u32) buffer index
82 @return - (vlib_buffer_t *) buffer pointer
84 always_inline vlib_buffer_t *
85 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
87 vlib_buffer_main_t *bm = vm->buffer_main;
90 b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
91 vlib_buffer_validate (vm, b);
95 static_always_inline u32
96 vlib_buffer_get_default_data_size (vlib_main_t * vm)
98 return vm->buffer_main->default_data_size;
101 static_always_inline void
102 vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
104 clib_memcpy_fast (dst, src, n_indices * sizeof (u32));
107 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
108 static_always_inline void
109 vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
111 #if defined CLIB_HAVE_VEC512
112 b->as_u8x64[0] = bt->as_u8x64[0];
113 #elif defined (CLIB_HAVE_VEC256)
114 b->as_u8x32[0] = bt->as_u8x32[0];
115 b->as_u8x32[1] = bt->as_u8x32[1];
116 #elif defined (CLIB_HAVE_VEC128)
117 b->as_u8x16[0] = bt->as_u8x16[0];
118 b->as_u8x16[1] = bt->as_u8x16[1];
119 b->as_u8x16[2] = bt->as_u8x16[2];
120 b->as_u8x16[3] = bt->as_u8x16[3];
122 clib_memcpy_fast (b, bt, 64);
127 vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node)
129 ASSERT (numa_node < vm->buffer_main->n_numa_nodes);
133 /** \brief Translate array of buffer indices into buffer pointers with offset
135 @param vm - (vlib_main_t *) vlib main data structure pointer
136 @param bi - (u32 *) array of buffer indices
137 @param b - (void **) array to store buffer pointers
138 @param count - (uword) number of elements
139 @param offset - (i32) offset applied to each pointer
141 static_always_inline void
142 vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
145 uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
146 #ifdef CLIB_HAVE_VEC256
147 u64x4 off = u64x4_splat (buffer_mem_start + offset);
148 /* if count is not const, compiler will not unroll while loop
149 se we maintain two-in-parallel variant */
152 u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
153 u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
154 /* shift and add to get vlib_buffer_t pointer */
155 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
156 u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
164 #ifdef CLIB_HAVE_VEC256
165 u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
166 /* shift and add to get vlib_buffer_t pointer */
167 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
168 #elif defined (CLIB_HAVE_VEC128)
169 u64x2 off = u64x2_splat (buffer_mem_start + offset);
170 u32x4 bi4 = u32x4_load_unaligned (bi);
171 u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
172 #if defined (__aarch64__)
173 u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4);
175 bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
176 u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
178 u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
179 u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
181 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
182 b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
183 b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
184 b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
192 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
199 /** \brief Translate array of buffer indices into buffer pointers
201 @param vm - (vlib_main_t *) vlib main data structure pointer
202 @param bi - (u32 *) array of buffer indices
203 @param b - (vlib_buffer_t **) array to store buffer pointers
204 @param count - (uword) number of elements
207 static_always_inline void
208 vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
210 vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
213 /** \brief Translate buffer pointer into buffer index
215 @param vm - (vlib_main_t *) vlib main data structure pointer
216 @param p - (void *) buffer pointer
217 @return - (u32) buffer index
221 vlib_get_buffer_index (vlib_main_t * vm, void *p)
223 vlib_buffer_main_t *bm = vm->buffer_main;
224 uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
225 ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
226 ASSERT (offset < bm->buffer_mem_size);
227 ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
228 return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
231 /** \brief Translate array of buffer pointers into buffer indices with offset
233 @param vm - (vlib_main_t *) vlib main data structure pointer
234 @param b - (void **) array of buffer pointers
235 @param bi - (u32 *) array to store buffer indices
236 @param count - (uword) number of elements
237 @param offset - (i32) offset applied to each pointer
239 static_always_inline void
240 vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
241 uword count, i32 offset)
243 #ifdef CLIB_HAVE_VEC256
244 u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
245 u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
249 /* load 4 pointers into 256-bit register */
250 u64x4 v0 = u64x4_load_unaligned (b);
251 u64x4 v1 = u64x4_load_unaligned (b + 4);
257 v0 >>= CLIB_LOG2_CACHE_LINE_BYTES;
258 v1 >>= CLIB_LOG2_CACHE_LINE_BYTES;
260 /* permute 256-bit register so lower u32s of each buffer index are
261 * placed into lower 128-bits */
262 v2 = u32x8_permute ((u32x8) v0, mask);
263 v3 = u32x8_permute ((u32x8) v1, mask);
265 /* extract lower 128-bits and save them to the array of buffer indices */
266 u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
267 u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
275 /* equivalent non-nector implementation */
276 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
277 bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
278 bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
279 bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
286 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
293 /** \brief Translate array of buffer pointers into buffer indices
295 @param vm - (vlib_main_t *) vlib main data structure pointer
296 @param b - (vlib_buffer_t **) array of buffer pointers
297 @param bi - (u32 *) array to store buffer indices
298 @param count - (uword) number of elements
300 static_always_inline void
301 vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi,
304 vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
307 /** \brief Get next buffer in buffer linklist, or zero for end of list.
309 @param vm - (vlib_main_t *) vlib main data structure pointer
310 @param b - (void *) buffer pointer
311 @return - (vlib_buffer_t *) next buffer, or NULL
313 always_inline vlib_buffer_t *
314 vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
316 return (b->flags & VLIB_BUFFER_NEXT_PRESENT
317 ? vlib_get_buffer (vm, b->next_buffer) : 0);
320 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
321 vlib_buffer_t * b_first);
323 /** \brief Get length in bytes of the buffer chain
325 @param vm - (vlib_main_t *) vlib main data structure pointer
326 @param b - (void *) buffer pointer
327 @return - (uword) length of buffer chain
330 vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
332 uword len = b->current_length;
334 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
337 if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
338 return len + b->total_length_not_including_first_buffer;
340 return vlib_buffer_length_in_chain_slow_path (vm, b);
343 /** \brief Get length in bytes of the buffer index buffer chain
345 @param vm - (vlib_main_t *) vlib main data structure pointer
346 @param bi - (u32) buffer index
347 @return - (uword) length of buffer chain
350 vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
352 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
353 return vlib_buffer_length_in_chain (vm, b);
356 /** \brief Copy buffer contents to memory
358 @param vm - (vlib_main_t *) vlib main data structure pointer
359 @param buffer_index - (u32) buffer index
360 @param contents - (u8 *) memory, <strong>must be large enough</strong>
361 @return - (uword) length of buffer chain
364 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
366 uword content_len = 0;
372 b = vlib_get_buffer (vm, buffer_index);
373 l = b->current_length;
374 clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
376 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
378 buffer_index = b->next_buffer;
385 vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
387 return vlib_physmem_get_pa (vm, b->data);
391 vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
393 return vlib_buffer_get_pa (vm, b) + b->current_data;
396 /** \brief Prefetch buffer metadata by buffer index
397 The first 64 bytes of buffer contains most header information
399 @param vm - (vlib_main_t *) vlib main data structure pointer
400 @param bi - (u32) buffer index
401 @param type - LOAD, STORE. In most cases, STORE is the right answer
403 /* Prefetch buffer header given index. */
404 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
406 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
407 vlib_prefetch_buffer_header (_b, type); \
412 /* Index is unknown. */
415 /* Index is known and free/allocated. */
416 VLIB_BUFFER_KNOWN_FREE,
417 VLIB_BUFFER_KNOWN_ALLOCATED,
418 } vlib_buffer_known_state_t;
420 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
422 vlib_buffer_known_state_t
425 always_inline vlib_buffer_known_state_t
426 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
428 vlib_buffer_main_t *bm = vm->buffer_main;
430 clib_spinlock_lock (&bm->buffer_known_hash_lockp);
431 uword *p = hash_get (bm->buffer_known_hash, buffer_index);
432 clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
433 return p ? p[0] : VLIB_BUFFER_UNKNOWN;
436 /* Validates sanity of a single buffer.
437 Returns format'ed vector with error message if any. */
438 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
441 static_always_inline vlib_buffer_pool_t *
442 vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
444 vlib_buffer_main_t *bm = vm->buffer_main;
445 return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
448 static_always_inline uword
449 vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
452 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
455 ASSERT (bp->buffers);
457 clib_spinlock_lock (&bp->lock);
458 len = vec_len (bp->buffers);
459 if (PREDICT_TRUE (n_buffers < len))
462 vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
463 _vec_len (bp->buffers) = len;
464 clib_spinlock_unlock (&bp->lock);
469 vlib_buffer_copy_indices (buffers, bp->buffers, len);
470 _vec_len (bp->buffers) = 0;
471 clib_spinlock_unlock (&bp->lock);
477 /** \brief Allocate buffers from specific pool into supplied array
479 @param vm - (vlib_main_t *) vlib main data structure pointer
480 @param buffers - (u32 * ) buffer index array
481 @param n_buffers - (u32) number of buffers requested
482 @return - (u32) number of buffers actually allocated, may be
483 less than the number requested or zero
487 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
488 u8 buffer_pool_index)
490 vlib_buffer_main_t *bm = vm->buffer_main;
491 vlib_buffer_pool_t *bp;
492 vlib_buffer_pool_thread_t *bpt;
493 u32 *src, *dst, len, n_left;
495 bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
496 bpt = vec_elt_at_index (bp->threads, vm->thread_index);
500 len = vec_len (bpt->cached_buffers);
502 /* per-thread cache contains enough buffers */
503 if (len >= n_buffers)
505 src = bpt->cached_buffers + len - n_buffers;
506 vlib_buffer_copy_indices (dst, src, n_buffers);
507 _vec_len (bpt->cached_buffers) -= n_buffers;
510 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
511 VLIB_BUFFER_KNOWN_FREE);
515 /* take everything available in the cache */
518 vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
519 _vec_len (bpt->cached_buffers) = 0;
524 len = round_pow2 (n_left, 32);
525 vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES);
526 len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
528 _vec_len (bpt->cached_buffers) = len;
532 u32 n_copy = clib_min (len, n_left);
533 src = bpt->cached_buffers + len - n_copy;
534 vlib_buffer_copy_indices (dst, src, n_copy);
535 _vec_len (bpt->cached_buffers) -= n_copy;
541 /* Verify that buffers are known free. */
543 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
544 VLIB_BUFFER_KNOWN_FREE);
549 /** \brief Allocate buffers from specific numa node into supplied array
551 @param vm - (vlib_main_t *) vlib main data structure pointer
552 @param buffers - (u32 * ) buffer index array
553 @param n_buffers - (u32) number of buffers requested
554 @param numa_node - (u32) numa node
555 @return - (u32) number of buffers actually allocated, may be
556 less than the number requested or zero
559 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
562 u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
563 return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
566 /** \brief Allocate buffers into supplied array
568 @param vm - (vlib_main_t *) vlib main data structure pointer
569 @param buffers - (u32 * ) buffer index array
570 @param n_buffers - (u32) number of buffers requested
571 @return - (u32) number of buffers actually allocated, may be
572 less than the number requested or zero
576 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
578 return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
581 /** \brief Allocate buffers into ring
583 @param vm - (vlib_main_t *) vlib main data structure pointer
584 @param buffers - (u32 * ) buffer index ring
585 @param start - (u32) first slot in the ring
586 @param ring_size - (u32) ring size
587 @param n_buffers - (u32) number of buffers requested
588 @return - (u32) number of buffers actually allocated, may be
589 less than the number requested or zero
592 vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
593 u32 ring_size, u32 n_buffers)
597 ASSERT (n_buffers <= ring_size);
599 if (PREDICT_TRUE (start + n_buffers <= ring_size))
600 return vlib_buffer_alloc (vm, ring + start, n_buffers);
602 n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
604 if (PREDICT_TRUE (n_alloc == ring_size - start))
605 n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
610 /** \brief Allocate buffers into ring from specific buffer pool
612 @param vm - (vlib_main_t *) vlib main data structure pointer
613 @param buffers - (u32 * ) buffer index ring
614 @param start - (u32) first slot in the ring
615 @param ring_size - (u32) ring size
616 @param n_buffers - (u32) number of buffers requested
617 @return - (u32) number of buffers actually allocated, may be
618 less than the number requested or zero
621 vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start,
622 u32 ring_size, u32 n_buffers,
623 u8 buffer_pool_index)
627 ASSERT (n_buffers <= ring_size);
629 if (PREDICT_TRUE (start + n_buffers <= ring_size))
630 return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
633 n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
636 if (PREDICT_TRUE (n_alloc == ring_size - start))
637 n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
643 static_always_inline void
644 vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
645 u32 * buffers, u32 n_buffers)
647 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
648 vlib_buffer_pool_thread_t *bpt =
649 vec_elt_at_index (bp->threads, vm->thread_index);
652 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
653 VLIB_BUFFER_KNOWN_ALLOCATED);
655 vec_add_aligned (bpt->cached_buffers, buffers, n_buffers,
656 CLIB_CACHE_LINE_BYTES);
658 if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE)
660 clib_spinlock_lock (&bp->lock);
661 /* keep last stored buffers, as they are more likely hot in the cache */
662 vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE,
663 CLIB_CACHE_LINE_BYTES);
664 vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0);
665 bpt->n_alloc -= VLIB_FRAME_SIZE;
666 clib_spinlock_unlock (&bp->lock);
670 static_always_inline void
671 vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
674 const int queue_size = 128;
675 vlib_buffer_pool_t *bp = 0;
676 u8 buffer_pool_index = ~0;
677 u32 n_queue = 0, queue[queue_size + 4];
678 vlib_buffer_t bt = { };
679 #if defined(CLIB_HAVE_VEC128) && !__aarch64__
680 vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
681 vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
682 vlib_buffer_t flags_refs_mask = {
683 .flags = VLIB_BUFFER_NEXT_PRESENT,
691 u32 bi, sum = 0, flags, next;
696 vlib_get_buffers (vm, buffers, b, 4);
697 vlib_get_buffers (vm, buffers + 8, b + 4, 4);
699 vlib_prefetch_buffer_header (b[4], LOAD);
700 vlib_prefetch_buffer_header (b[5], LOAD);
701 vlib_prefetch_buffer_header (b[6], LOAD);
702 vlib_prefetch_buffer_header (b[7], LOAD);
704 #if defined(CLIB_HAVE_VEC128) && !__aarch64__
705 u8x16 p0, p1, p2, p3, r;
706 p0 = u8x16_load_unaligned (b[0]);
707 p1 = u8x16_load_unaligned (b[1]);
708 p2 = u8x16_load_unaligned (b[2]);
709 p3 = u8x16_load_unaligned (b[3]);
711 r = p0 ^ bpi_vec.as_u8x16[0];
712 r |= p1 ^ bpi_vec.as_u8x16[0];
713 r |= p2 ^ bpi_vec.as_u8x16[0];
714 r |= p3 ^ bpi_vec.as_u8x16[0];
715 r &= bpi_mask.as_u8x16[0];
716 r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
718 sum = !u8x16_is_all_zero (r);
724 sum &= VLIB_BUFFER_NEXT_PRESENT;
725 sum += b[0]->ref_count - 1;
726 sum += b[1]->ref_count - 1;
727 sum += b[2]->ref_count - 1;
728 sum += b[3]->ref_count - 1;
729 sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
730 sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
731 sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
732 sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
738 vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
739 vlib_buffer_copy_template (b[0], &bt);
740 vlib_buffer_copy_template (b[1], &bt);
741 vlib_buffer_copy_template (b[2], &bt);
742 vlib_buffer_copy_template (b[3], &bt);
745 vlib_buffer_validate (vm, b[0]);
746 vlib_buffer_validate (vm, b[1]);
747 vlib_buffer_validate (vm, b[2]);
748 vlib_buffer_validate (vm, b[3]);
750 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
751 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
752 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
753 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
755 if (n_queue >= queue_size)
757 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
768 b[0] = vlib_get_buffer (vm, bi);
770 next = b[0]->next_buffer;
772 if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
777 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
781 buffer_pool_index = b[0]->buffer_pool_index;
782 #if defined(CLIB_HAVE_VEC128) && !__aarch64__
783 bpi_vec.buffer_pool_index = buffer_pool_index;
785 bp = vlib_get_buffer_pool (vm, buffer_pool_index);
786 vlib_buffer_copy_template (&bt, &bp->buffer_template);
789 vlib_buffer_validate (vm, b[0]);
791 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
793 if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
795 vlib_buffer_copy_template (b[0], &bt);
796 queue[n_queue++] = bi;
799 if (n_queue == queue_size)
801 vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
805 if (flags & VLIB_BUFFER_NEXT_PRESENT)
816 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
820 /** \brief Free buffers
821 Frees the entire buffer chain for each buffer
823 @param vm - (vlib_main_t *) vlib main data structure pointer
824 @param buffers - (u32 * ) buffer index array
825 @param n_buffers - (u32) number of buffers to free
829 vlib_buffer_free (vlib_main_t * vm,
830 /* pointer to first buffer */
832 /* number of buffers to free */
835 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
838 /** \brief Free buffers, does not free the buffer chain for each buffer
840 @param vm - (vlib_main_t *) vlib main data structure pointer
841 @param buffers - (u32 * ) buffer index array
842 @param n_buffers - (u32) number of buffers to free
846 vlib_buffer_free_no_next (vlib_main_t * vm,
847 /* pointer to first buffer */
849 /* number of buffers to free */
852 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
855 /** \brief Free one buffer
856 Shorthand to free a single buffer chain.
858 @param vm - (vlib_main_t *) vlib main data structure pointer
859 @param buffer_index - (u32) buffer index to free
862 vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
864 vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
867 /** \brief Free buffers from ring
869 @param vm - (vlib_main_t *) vlib main data structure pointer
870 @param buffers - (u32 * ) buffer index ring
871 @param start - (u32) first slot in the ring
872 @param ring_size - (u32) ring size
873 @param n_buffers - (u32) number of buffers
876 vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start,
877 u32 ring_size, u32 n_buffers)
879 ASSERT (n_buffers <= ring_size);
881 if (PREDICT_TRUE (start + n_buffers <= ring_size))
883 vlib_buffer_free (vm, ring + start, n_buffers);
887 vlib_buffer_free (vm, ring + start, ring_size - start);
888 vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
892 /** \brief Free buffers from ring without freeing tail buffers
894 @param vm - (vlib_main_t *) vlib main data structure pointer
895 @param buffers - (u32 * ) buffer index ring
896 @param start - (u32) first slot in the ring
897 @param ring_size - (u32) ring size
898 @param n_buffers - (u32) number of buffers
901 vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
902 u32 ring_size, u32 n_buffers)
904 ASSERT (n_buffers <= ring_size);
906 if (PREDICT_TRUE (start + n_buffers <= ring_size))
908 vlib_buffer_free_no_next (vm, ring + start, n_buffers);
912 vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
913 vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
917 /* Append given data to end of buffer, possibly allocating new buffers. */
918 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
921 /* duplicate all buffers in chain */
922 always_inline vlib_buffer_t *
923 vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
925 vlib_buffer_t *s, *d, *fd;
926 uword n_alloc, n_buffers = 1;
927 u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
931 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
934 s = vlib_get_buffer (vm, s->next_buffer);
936 u32 new_buffers[n_buffers];
938 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
940 /* No guarantee that we'll get all the buffers we asked for */
941 if (PREDICT_FALSE (n_alloc < n_buffers))
944 vlib_buffer_free (vm, new_buffers, n_alloc);
950 fd = d = vlib_get_buffer (vm, new_buffers[0]);
951 d->current_data = s->current_data;
952 d->current_length = s->current_length;
953 d->flags = s->flags & flag_mask;
954 d->total_length_not_including_first_buffer =
955 s->total_length_not_including_first_buffer;
956 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
957 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
958 clib_memcpy_fast (vlib_buffer_get_current (d),
959 vlib_buffer_get_current (s), s->current_length);
962 for (i = 1; i < n_buffers; i++)
965 d->next_buffer = new_buffers[i];
967 s = vlib_get_buffer (vm, s->next_buffer);
968 d = vlib_get_buffer (vm, new_buffers[i]);
969 d->current_data = s->current_data;
970 d->current_length = s->current_length;
971 clib_memcpy_fast (vlib_buffer_get_current (d),
972 vlib_buffer_get_current (s), s->current_length);
973 d->flags = s->flags & flag_mask;
979 /** \brief Create a maximum of 256 clones of buffer and store them
980 in the supplied array
982 @param vm - (vlib_main_t *) vlib main data structure pointer
983 @param src_buffer - (u32) source buffer index
984 @param buffers - (u32 * ) buffer index array
985 @param n_buffers - (u16) number of buffer clones requested (<=256)
986 @param head_end_offset - (u16) offset relative to current position
987 where packet head ends
988 @return - (u16) number of buffers actually cloned, may be
989 less than the number requested or zero
992 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
993 u16 n_buffers, u16 head_end_offset)
996 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
998 ASSERT (s->ref_count == 1);
1000 ASSERT (n_buffers <= 256);
1002 if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
1004 buffers[0] = src_buffer;
1005 for (i = 1; i < n_buffers; i++)
1008 d = vlib_buffer_copy (vm, s);
1011 buffers[i] = vlib_get_buffer_index (vm, d);
1017 if (PREDICT_FALSE (n_buffers == 1))
1019 buffers[0] = src_buffer;
1023 n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
1024 s->buffer_pool_index);
1026 for (i = 0; i < n_buffers; i++)
1028 vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
1029 d->current_data = s->current_data;
1030 d->current_length = head_end_offset;
1031 ASSERT (d->buffer_pool_index == s->buffer_pool_index);
1033 d->total_length_not_including_first_buffer = s->current_length -
1035 if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
1037 d->total_length_not_including_first_buffer +=
1038 s->total_length_not_including_first_buffer;
1040 d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
1041 d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1042 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1043 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1044 clib_memcpy_fast (vlib_buffer_get_current (d),
1045 vlib_buffer_get_current (s), head_end_offset);
1046 d->next_buffer = src_buffer;
1048 vlib_buffer_advance (s, head_end_offset);
1049 s->ref_count = n_buffers;
1050 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1052 s = vlib_get_buffer (vm, s->next_buffer);
1053 s->ref_count = n_buffers;
1059 /** \brief Create multiple clones of buffer and store them
1060 in the supplied array
1062 @param vm - (vlib_main_t *) vlib main data structure pointer
1063 @param src_buffer - (u32) source buffer index
1064 @param buffers - (u32 * ) buffer index array
1065 @param n_buffers - (u16) number of buffer clones requested (<=256)
1066 @param head_end_offset - (u16) offset relative to current position
1067 where packet head ends
1068 @return - (u16) number of buffers actually cloned, may be
1069 less than the number requested or zero
1072 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1073 u16 n_buffers, u16 head_end_offset)
1075 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1078 while (n_buffers > 256)
1080 vlib_buffer_t *copy;
1081 copy = vlib_buffer_copy (vm, s);
1082 n_cloned += vlib_buffer_clone_256 (vm,
1083 vlib_get_buffer_index (vm, copy),
1084 (buffers + n_cloned),
1085 256, head_end_offset);
1088 n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
1090 n_buffers, head_end_offset);
1095 /** \brief Attach cloned tail to the buffer
1097 @param vm - (vlib_main_t *) vlib main data structure pointer
1098 @param head - (vlib_buffer_t *) head buffer
1099 @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
1103 vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
1104 vlib_buffer_t * tail)
1106 ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1107 ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
1109 head->flags |= VLIB_BUFFER_NEXT_PRESENT;
1110 head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1111 head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1112 head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1113 head->next_buffer = vlib_get_buffer_index (vm, tail);
1114 head->total_length_not_including_first_buffer = tail->current_length +
1115 tail->total_length_not_including_first_buffer;
1118 clib_atomic_add_fetch (&tail->ref_count, 1);
1120 if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
1122 tail = vlib_get_buffer (vm, tail->next_buffer);
1127 /* Initializes the buffer as an empty packet with no chained buffers. */
1129 vlib_buffer_chain_init (vlib_buffer_t * first)
1131 first->total_length_not_including_first_buffer = 0;
1132 first->current_length = 0;
1133 first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1134 first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1137 /* The provided next_bi buffer index is appended to the end of the packet. */
1138 always_inline vlib_buffer_t *
1139 vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
1141 vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
1142 last->next_buffer = next_bi;
1143 last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1144 next_buffer->current_length = 0;
1145 next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1149 /* Increases or decreases the packet length.
1150 * It does not allocate or deallocate new buffers.
1151 * Therefore, the added length must be compatible
1152 * with the last buffer. */
1154 vlib_buffer_chain_increase_length (vlib_buffer_t * first,
1155 vlib_buffer_t * last, i32 len)
1157 last->current_length += len;
1159 first->total_length_not_including_first_buffer += len;
1162 /* Copy data to the end of the packet and increases its length.
1163 * It does not allocate new buffers.
1164 * Returns the number of copied bytes. */
1166 vlib_buffer_chain_append_data (vlib_main_t * vm,
1167 vlib_buffer_t * first,
1168 vlib_buffer_t * last, void *data, u16 data_len)
1170 u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
1171 ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
1172 u16 len = clib_min (data_len,
1173 n_buffer_bytes - last->current_length -
1174 last->current_data);
1175 clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
1177 vlib_buffer_chain_increase_length (first, last, len);
1181 /* Copy data to the end of the packet and increases its length.
1182 * Allocates additional buffers from the free list if necessary.
1183 * Returns the number of copied bytes.
1184 * 'last' value is modified whenever new buffers are allocated and
1185 * chained and points to the last buffer in the chain. */
1187 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
1188 vlib_buffer_t * first,
1189 vlib_buffer_t ** last, void *data,
1191 void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);
1193 format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
1194 format_vlib_buffer_contents;
1198 /* Vector of packet data. */
1201 /* Number of buffers to allocate in each call to allocator. */
1202 u32 min_n_buffers_each_alloc;
1205 } vlib_packet_template_t;
1207 void vlib_packet_template_init (vlib_main_t * vm,
1208 vlib_packet_template_t * t,
1210 uword n_packet_data_bytes,
1211 uword min_n_buffers_each_alloc,
1214 void *vlib_packet_template_get_packet (vlib_main_t * vm,
1215 vlib_packet_template_t * t,
1219 vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
1221 vec_free (t->packet_data);
1225 vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b)
1227 return b->data + vlib_buffer_get_default_data_size (vm) -
1228 ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1232 vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
1234 vlib_buffer_t *db = b, *sb, *first = b;
1236 u32 bytes_left = 0, data_size;
1237 u16 src_left, dst_left, n_buffers = 1;
1241 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1244 data_size = vlib_buffer_get_default_data_size (vm);
1246 dst_left = vlib_buffer_space_left_at_end (vm, b);
1248 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1250 b = vlib_get_buffer (vm, b->next_buffer);
1251 if (b->ref_count > 1)
1253 bytes_left += b->current_length;
1257 /* if buffer is cloned, create completely new chain - unless everything fits
1258 * into one buffer */
1259 if (is_cloned && bytes_left >= dst_left)
1262 u32 space_needed = bytes_left - dst_left;
1265 if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1270 b = vlib_get_buffer (vm, tail);
1272 while (len < space_needed)
1275 if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1277 vlib_buffer_free_one (vm, tail);
1280 b->flags = VLIB_BUFFER_NEXT_PRESENT;
1281 b->next_buffer = bi;
1282 b = vlib_get_buffer (vm, bi);
1286 sb = vlib_get_buffer (vm, first->next_buffer);
1287 to_free = first->next_buffer;
1288 first->next_buffer = tail;
1291 sb = vlib_get_buffer (vm, first->next_buffer);
1293 src_left = sb->current_length;
1294 sp = vlib_buffer_get_current (sb);
1295 dp = vlib_buffer_get_tail (db);
1304 db->current_data = 0;
1305 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1306 ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1307 db = vlib_get_buffer (vm, db->next_buffer);
1308 dst_left = data_size;
1312 while (src_left == 0)
1314 ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1315 sb = vlib_get_buffer (vm, sb->next_buffer);
1316 src_left = sb->current_length;
1317 sp = vlib_buffer_get_current (sb);
1320 bytes_to_copy = clib_min (dst_left, src_left);
1325 bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1327 clib_memcpy_fast (dp, sp, bytes_to_copy);
1330 src_left -= bytes_to_copy;
1331 dst_left -= bytes_to_copy;
1332 dp += bytes_to_copy;
1333 sp += bytes_to_copy;
1334 bytes_left -= bytes_to_copy;
1337 db->current_data = 0;
1338 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1340 if (is_cloned && to_free)
1341 vlib_buffer_free_one (vm, to_free);
1344 if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1345 vlib_buffer_free_one (vm, db->next_buffer);
1346 db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1349 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1351 b = vlib_get_buffer (vm, b->next_buffer);
1356 first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1361 #endif /* included_vlib_buffer_funcs_h */
1364 * fd.io coding-style-patch-verification: ON
1367 * eval: (c-set-style "gnu")