2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_funcs.h: VLIB buffer related functions/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
51 vlib buffer access methods.
54 typedef void (vlib_buffer_enqueue_to_next_fn_t) (vlib_main_t *vm,
55 vlib_node_runtime_t *node,
56 u32 *buffers, u16 *nexts,
58 typedef void (vlib_buffer_enqueue_to_single_next_fn_t) (
59 vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index,
62 typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) (
63 vlib_main_t *vm, u32 frame_queue_index, u32 *buffer_indices,
64 u16 *thread_indices, u32 n_packets, int drop_on_congestion);
67 vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn;
68 vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn;
69 vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn;
70 } vlib_buffer_func_main_t;
72 extern vlib_buffer_func_main_t vlib_buffer_func_main;
75 vlib_buffer_validate (vlib_main_t * vm, vlib_buffer_t * b)
77 vlib_buffer_main_t *bm = vm->buffer_main;
78 vlib_buffer_pool_t *bp;
80 /* reference count in allocated buffer always must be 1 or higher */
81 ASSERT (b->ref_count > 0);
83 /* verify that buffer pool index is valid */
84 bp = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index);
85 ASSERT (pointer_to_uword (b) >= bp->start);
86 ASSERT (pointer_to_uword (b) < bp->start + bp->size -
87 (bp->data_size + sizeof (vlib_buffer_t)));
91 vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
94 offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
95 return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
98 /** \brief Translate buffer index into buffer pointer
100 @param vm - (vlib_main_t *) vlib main data structure pointer
101 @param buffer_index - (u32) buffer index
102 @return - (vlib_buffer_t *) buffer pointer
104 always_inline vlib_buffer_t *
105 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
107 vlib_buffer_main_t *bm = vm->buffer_main;
110 b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
111 vlib_buffer_validate (vm, b);
115 static_always_inline u32
116 vlib_buffer_get_default_data_size (vlib_main_t * vm)
118 return vm->buffer_main->default_data_size;
121 static_always_inline void
122 vlib_buffer_copy_indices (u32 * dst, u32 * src, u32 n_indices)
124 #if defined(CLIB_HAVE_VEC512)
125 while (n_indices >= 16)
127 u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
134 #if defined(CLIB_HAVE_VEC256)
135 while (n_indices >= 8)
137 u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
144 #if defined(CLIB_HAVE_VEC128)
145 while (n_indices >= 4)
147 u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
164 vlib_buffer_copy_indices_from_ring (u32 * dst, u32 * ring, u32 start,
165 u32 ring_size, u32 n_buffers)
167 ASSERT (n_buffers <= ring_size);
169 if (PREDICT_TRUE (start + n_buffers <= ring_size))
171 vlib_buffer_copy_indices (dst, ring + start, n_buffers);
175 u32 n = ring_size - start;
176 vlib_buffer_copy_indices (dst, ring + start, n);
177 vlib_buffer_copy_indices (dst + n, ring, n_buffers - n);
182 vlib_buffer_copy_indices_to_ring (u32 * ring, u32 * src, u32 start,
183 u32 ring_size, u32 n_buffers)
185 ASSERT (n_buffers <= ring_size);
187 if (PREDICT_TRUE (start + n_buffers <= ring_size))
189 vlib_buffer_copy_indices (ring + start, src, n_buffers);
193 u32 n = ring_size - start;
194 vlib_buffer_copy_indices (ring + start, src, n);
195 vlib_buffer_copy_indices (ring, src + n, n_buffers - n);
199 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
200 static_always_inline void
201 vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
203 #if defined CLIB_HAVE_VEC512
204 b->as_u8x64[0] = bt->as_u8x64[0];
205 #elif defined (CLIB_HAVE_VEC256)
206 b->as_u8x32[0] = bt->as_u8x32[0];
207 b->as_u8x32[1] = bt->as_u8x32[1];
208 #elif defined (CLIB_HAVE_VEC128)
209 b->as_u8x16[0] = bt->as_u8x16[0];
210 b->as_u8x16[1] = bt->as_u8x16[1];
211 b->as_u8x16[2] = bt->as_u8x16[2];
212 b->as_u8x16[3] = bt->as_u8x16[3];
214 clib_memcpy_fast (b, bt, 64);
219 vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node)
221 ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
222 return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
225 /** \brief Translate array of buffer indices into buffer pointers with offset
227 @param vm - (vlib_main_t *) vlib main data structure pointer
228 @param bi - (u32 *) array of buffer indices
229 @param b - (void **) array to store buffer pointers
230 @param count - (uword) number of elements
231 @param offset - (i32) offset applied to each pointer
233 static_always_inline void
234 vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
237 uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
238 #ifdef CLIB_HAVE_VEC512
239 u64x8 of8 = u64x8_splat (buffer_mem_start + offset);
240 u64x4 off = u64x8_extract_lo (of8);
241 /* if count is not const, compiler will not unroll while loop
242 se we maintain two-in-parallel variant */
245 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
246 u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8));
247 u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16));
248 u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24));
249 /* shift and add to get vlib_buffer_t pointer */
250 u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b);
251 u64x8_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 8);
252 u64x8_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 16);
253 u64x8_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b + 24);
260 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
261 /* shift and add to get vlib_buffer_t pointer */
262 u64x8_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + of8, b);
267 #elif defined CLIB_HAVE_VEC256
268 u64x4 off = u64x4_splat (buffer_mem_start + offset);
269 /* if count is not const, compiler will not unroll while loop
270 se we maintain two-in-parallel variant */
273 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
274 u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4));
275 u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8));
276 u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12));
277 u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16));
278 u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20));
279 u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24));
280 u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28));
281 /* shift and add to get vlib_buffer_t pointer */
282 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
283 u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
284 u64x4_store_unaligned ((b2 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 8);
285 u64x4_store_unaligned ((b3 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 12);
286 u64x4_store_unaligned ((b4 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 16);
287 u64x4_store_unaligned ((b5 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 20);
288 u64x4_store_unaligned ((b6 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 24);
289 u64x4_store_unaligned ((b7 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 28);
297 #ifdef CLIB_HAVE_VEC256
298 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
299 /* shift and add to get vlib_buffer_t pointer */
300 u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
301 #elif defined (CLIB_HAVE_VEC128)
302 u64x2 off = u64x2_splat (buffer_mem_start + offset);
303 u32x4 bi4 = u32x4_load_unaligned (bi);
304 u64x2 b0 = u64x2_from_u32x4 ((u32x4) bi4);
305 #if defined (__aarch64__)
306 u64x2 b1 = u64x2_from_u32x4_high ((u32x4) bi4);
308 bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
309 u64x2 b1 = u64x2_from_u32x4 ((u32x4) bi4);
311 u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
312 u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
314 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
315 b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
316 b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
317 b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
325 b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
332 /** \brief Translate array of buffer indices into buffer pointers
334 @param vm - (vlib_main_t *) vlib main data structure pointer
335 @param bi - (u32 *) array of buffer indices
336 @param b - (vlib_buffer_t **) array to store buffer pointers
337 @param count - (uword) number of elements
340 static_always_inline void
341 vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
343 vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
346 /** \brief Translate buffer pointer into buffer index
348 @param vm - (vlib_main_t *) vlib main data structure pointer
349 @param p - (void *) buffer pointer
350 @return - (u32) buffer index
354 vlib_get_buffer_index (vlib_main_t * vm, void *p)
356 vlib_buffer_main_t *bm = vm->buffer_main;
357 uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
358 ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
359 ASSERT (offset < bm->buffer_mem_size);
360 ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
361 return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
364 /** \brief Translate array of buffer pointers into buffer indices with offset
366 @param vm - (vlib_main_t *) vlib main data structure pointer
367 @param b - (void **) array of buffer pointers
368 @param bi - (u32 *) array to store buffer indices
369 @param count - (uword) number of elements
370 @param offset - (i32) offset applied to each pointer
372 static_always_inline void
373 vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
374 uword count, i32 offset)
376 #ifdef CLIB_HAVE_VEC256
377 u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
378 u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
382 /* load 4 pointers into 256-bit register */
383 u64x4 v0 = u64x4_load_unaligned (b);
384 u64x4 v1 = u64x4_load_unaligned (b + 4);
390 v0 >>= CLIB_LOG2_CACHE_LINE_BYTES;
391 v1 >>= CLIB_LOG2_CACHE_LINE_BYTES;
393 /* permute 256-bit register so lower u32s of each buffer index are
394 * placed into lower 128-bits */
395 v2 = u32x8_permute ((u32x8) v0, mask);
396 v3 = u32x8_permute ((u32x8) v1, mask);
398 /* extract lower 128-bits and save them to the array of buffer indices */
399 u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
400 u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
408 /* equivalent non-nector implementation */
409 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
410 bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
411 bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
412 bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
419 bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
426 /** \brief Translate array of buffer pointers into buffer indices
428 @param vm - (vlib_main_t *) vlib main data structure pointer
429 @param b - (vlib_buffer_t **) array of buffer pointers
430 @param bi - (u32 *) array to store buffer indices
431 @param count - (uword) number of elements
433 static_always_inline void
434 vlib_get_buffer_indices (vlib_main_t * vm, vlib_buffer_t ** b, u32 * bi,
437 vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
440 /** \brief Get next buffer in buffer linklist, or zero for end of list.
442 @param vm - (vlib_main_t *) vlib main data structure pointer
443 @param b - (void *) buffer pointer
444 @return - (vlib_buffer_t *) next buffer, or NULL
446 always_inline vlib_buffer_t *
447 vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
449 return (b->flags & VLIB_BUFFER_NEXT_PRESENT
450 ? vlib_get_buffer (vm, b->next_buffer) : 0);
453 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
454 vlib_buffer_t * b_first);
456 /** \brief Get length in bytes of the buffer chain
458 @param vm - (vlib_main_t *) vlib main data structure pointer
459 @param b - (void *) buffer pointer
460 @return - (uword) length of buffer chain
463 vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
465 uword len = b->current_length;
467 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
470 if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
471 return len + b->total_length_not_including_first_buffer;
473 return vlib_buffer_length_in_chain_slow_path (vm, b);
476 /** \brief Get length in bytes of the buffer index buffer chain
478 @param vm - (vlib_main_t *) vlib main data structure pointer
479 @param bi - (u32) buffer index
480 @return - (uword) length of buffer chain
483 vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
485 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
486 return vlib_buffer_length_in_chain (vm, b);
489 /** \brief Copy buffer contents to memory
491 @param vm - (vlib_main_t *) vlib main data structure pointer
492 @param buffer_index - (u32) buffer index
493 @param contents - (u8 *) memory, <strong>must be large enough</strong>
494 @return - (uword) length of buffer chain
497 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
499 uword content_len = 0;
505 b = vlib_get_buffer (vm, buffer_index);
506 l = b->current_length;
507 clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
509 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
511 buffer_index = b->next_buffer;
518 vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
520 return vlib_physmem_get_pa (vm, b->data);
524 vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
526 return vlib_buffer_get_pa (vm, b) + b->current_data;
529 /** \brief Prefetch buffer metadata by buffer index
530 The first 64 bytes of buffer contains most header information
532 @param vm - (vlib_main_t *) vlib main data structure pointer
533 @param bi - (u32) buffer index
534 @param type - LOAD, STORE. In most cases, STORE is the right answer
536 /* Prefetch buffer header given index. */
537 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
539 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
540 vlib_prefetch_buffer_header (_b, type); \
545 /* Index is unknown. */
548 /* Index is known and free/allocated. */
549 VLIB_BUFFER_KNOWN_FREE,
550 VLIB_BUFFER_KNOWN_ALLOCATED,
551 } vlib_buffer_known_state_t;
553 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
555 vlib_buffer_known_state_t
558 always_inline vlib_buffer_known_state_t
559 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
561 vlib_buffer_main_t *bm = vm->buffer_main;
563 clib_spinlock_lock (&bm->buffer_known_hash_lockp);
564 uword *p = hash_get (bm->buffer_known_hash, buffer_index);
565 clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
566 return p ? p[0] : VLIB_BUFFER_UNKNOWN;
569 /* Validates sanity of a single buffer.
570 Returns format'ed vector with error message if any. */
571 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
574 u8 *vlib_validate_buffers (vlib_main_t * vm,
576 uword next_buffer_stride,
578 vlib_buffer_known_state_t known_state,
579 uword follow_buffer_next);
581 static_always_inline vlib_buffer_pool_t *
582 vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
584 vlib_buffer_main_t *bm = vm->buffer_main;
585 return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
588 static_always_inline __clib_warn_unused_result uword
589 vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
592 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
595 ASSERT (bp->buffers);
597 clib_spinlock_lock (&bp->lock);
599 if (PREDICT_TRUE (n_buffers < len))
602 vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
604 clib_spinlock_unlock (&bp->lock);
609 vlib_buffer_copy_indices (buffers, bp->buffers, len);
611 clib_spinlock_unlock (&bp->lock);
617 /** \brief Allocate buffers from specific pool into supplied array
619 @param vm - (vlib_main_t *) vlib main data structure pointer
620 @param buffers - (u32 * ) buffer index array
621 @param n_buffers - (u32) number of buffers requested
622 @return - (u32) number of buffers actually allocated, may be
623 less than the number requested or zero
626 always_inline __clib_warn_unused_result u32
627 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
628 u8 buffer_pool_index)
630 vlib_buffer_main_t *bm = vm->buffer_main;
631 vlib_buffer_pool_t *bp;
632 vlib_buffer_pool_thread_t *bpt;
633 u32 *src, *dst, len, n_left;
635 /* If buffer allocation fault injection is configured */
636 if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
638 u32 vlib_buffer_alloc_may_fail (vlib_main_t *, u32);
640 /* See how many buffers we're willing to allocate */
641 n_buffers = vlib_buffer_alloc_may_fail (vm, n_buffers);
646 bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
647 bpt = vec_elt_at_index (bp->threads, vm->thread_index);
653 /* per-thread cache contains enough buffers */
654 if (len >= n_buffers)
656 src = bpt->cached_buffers + len - n_buffers;
657 vlib_buffer_copy_indices (dst, src, n_buffers);
658 bpt->n_cached -= n_buffers;
661 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
662 VLIB_BUFFER_KNOWN_FREE);
666 /* alloc bigger than cache - take buffers directly from main pool */
667 if (n_buffers >= VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ)
669 n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers,
673 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
674 VLIB_BUFFER_KNOWN_FREE);
678 /* take everything available in the cache */
681 vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
687 len = round_pow2 (n_left, 32);
688 len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
694 u32 n_copy = clib_min (len, n_left);
695 src = bpt->cached_buffers + len - n_copy;
696 vlib_buffer_copy_indices (dst, src, n_copy);
697 bpt->n_cached -= n_copy;
703 /* Verify that buffers are known free. */
705 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
706 VLIB_BUFFER_KNOWN_FREE);
711 /** \brief Allocate buffers from specific numa node into supplied array
713 @param vm - (vlib_main_t *) vlib main data structure pointer
714 @param buffers - (u32 * ) buffer index array
715 @param n_buffers - (u32) number of buffers requested
716 @param numa_node - (u32) numa node
717 @return - (u32) number of buffers actually allocated, may be
718 less than the number requested or zero
720 always_inline __clib_warn_unused_result u32
721 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
724 u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
725 return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
728 /** \brief Allocate buffers into supplied array
730 @param vm - (vlib_main_t *) vlib main data structure pointer
731 @param buffers - (u32 * ) buffer index array
732 @param n_buffers - (u32) number of buffers requested
733 @return - (u32) number of buffers actually allocated, may be
734 less than the number requested or zero
737 always_inline __clib_warn_unused_result u32
738 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
740 return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
743 /** \brief Allocate buffers into ring
745 @param vm - (vlib_main_t *) vlib main data structure pointer
746 @param buffers - (u32 * ) buffer index ring
747 @param start - (u32) first slot in the ring
748 @param ring_size - (u32) ring size
749 @param n_buffers - (u32) number of buffers requested
750 @return - (u32) number of buffers actually allocated, may be
751 less than the number requested or zero
753 always_inline __clib_warn_unused_result u32
754 vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start,
755 u32 ring_size, u32 n_buffers)
759 ASSERT (n_buffers <= ring_size);
761 if (PREDICT_TRUE (start + n_buffers <= ring_size))
762 return vlib_buffer_alloc (vm, ring + start, n_buffers);
764 n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
766 if (PREDICT_TRUE (n_alloc == ring_size - start))
767 n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
772 /** \brief Allocate buffers into ring from specific buffer pool
774 @param vm - (vlib_main_t *) vlib main data structure pointer
775 @param buffers - (u32 * ) buffer index ring
776 @param start - (u32) first slot in the ring
777 @param ring_size - (u32) ring size
778 @param n_buffers - (u32) number of buffers requested
779 @return - (u32) number of buffers actually allocated, may be
780 less than the number requested or zero
782 always_inline __clib_warn_unused_result u32
783 vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start,
784 u32 ring_size, u32 n_buffers,
785 u8 buffer_pool_index)
789 ASSERT (n_buffers <= ring_size);
791 if (PREDICT_TRUE (start + n_buffers <= ring_size))
792 return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
795 n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
798 if (PREDICT_TRUE (n_alloc == ring_size - start))
799 n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
805 static_always_inline void
806 vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
807 u32 * buffers, u32 n_buffers)
809 vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
810 vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads,
812 u32 n_cached, n_empty;
815 vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
816 VLIB_BUFFER_KNOWN_ALLOCATED);
818 n_cached = bpt->n_cached;
819 n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached;
820 if (n_buffers <= n_empty)
822 vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
824 bpt->n_cached = n_cached + n_buffers;
828 vlib_buffer_copy_indices (bpt->cached_buffers + n_cached,
829 buffers + n_buffers - n_empty, n_empty);
830 bpt->n_cached = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ;
832 clib_spinlock_lock (&bp->lock);
833 vlib_buffer_copy_indices (bp->buffers + bp->n_avail, buffers,
834 n_buffers - n_empty);
835 bp->n_avail += n_buffers - n_empty;
836 clib_spinlock_unlock (&bp->lock);
839 static_always_inline void
840 vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
843 const int queue_size = 128;
844 vlib_buffer_pool_t *bp = 0;
845 u8 buffer_pool_index = ~0;
846 u32 n_queue = 0, queue[queue_size + 4];
847 vlib_buffer_t bt = { };
848 #if defined(CLIB_HAVE_VEC128)
849 vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
850 vlib_buffer_t bpi_vec = {};
851 vlib_buffer_t flags_refs_mask = {
852 .flags = VLIB_BUFFER_NEXT_PRESENT,
857 if (PREDICT_FALSE (n_buffers == 0))
860 vlib_buffer_t *b = vlib_get_buffer (vm, buffers[0]);
861 buffer_pool_index = b->buffer_pool_index;
862 bp = vlib_get_buffer_pool (vm, buffer_pool_index);
863 vlib_buffer_copy_template (&bt, &bp->buffer_template);
864 #if defined(CLIB_HAVE_VEC128)
865 bpi_vec.buffer_pool_index = buffer_pool_index;
871 u32 bi, sum = 0, flags, next;
876 vlib_get_buffers (vm, buffers, b, 4);
880 vlib_get_buffers (vm, buffers + 8, b + 4, 4);
881 vlib_prefetch_buffer_header (b[4], LOAD);
882 vlib_prefetch_buffer_header (b[5], LOAD);
883 vlib_prefetch_buffer_header (b[6], LOAD);
884 vlib_prefetch_buffer_header (b[7], LOAD);
887 #if defined(CLIB_HAVE_VEC128)
888 u8x16 p0, p1, p2, p3, r;
889 p0 = u8x16_load_unaligned (b[0]);
890 p1 = u8x16_load_unaligned (b[1]);
891 p2 = u8x16_load_unaligned (b[2]);
892 p3 = u8x16_load_unaligned (b[3]);
894 r = p0 ^ bpi_vec.as_u8x16[0];
895 r |= p1 ^ bpi_vec.as_u8x16[0];
896 r |= p2 ^ bpi_vec.as_u8x16[0];
897 r |= p3 ^ bpi_vec.as_u8x16[0];
898 r &= bpi_mask.as_u8x16[0];
899 r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
901 sum = !u8x16_is_all_zero (r);
907 sum &= VLIB_BUFFER_NEXT_PRESENT;
908 sum += b[0]->ref_count - 1;
909 sum += b[1]->ref_count - 1;
910 sum += b[2]->ref_count - 1;
911 sum += b[3]->ref_count - 1;
912 sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
913 sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
914 sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
915 sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
921 vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
922 vlib_buffer_copy_template (b[0], &bt);
923 vlib_buffer_copy_template (b[1], &bt);
924 vlib_buffer_copy_template (b[2], &bt);
925 vlib_buffer_copy_template (b[3], &bt);
928 vlib_buffer_validate (vm, b[0]);
929 vlib_buffer_validate (vm, b[1]);
930 vlib_buffer_validate (vm, b[2]);
931 vlib_buffer_validate (vm, b[3]);
933 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
934 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]);
935 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]);
936 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]);
938 if (n_queue >= queue_size)
940 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
951 b[0] = vlib_get_buffer (vm, bi);
953 next = b[0]->next_buffer;
955 if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
960 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
964 buffer_pool_index = b[0]->buffer_pool_index;
965 #if defined(CLIB_HAVE_VEC128)
966 bpi_vec.buffer_pool_index = buffer_pool_index;
968 bp = vlib_get_buffer_pool (vm, buffer_pool_index);
969 vlib_buffer_copy_template (&bt, &bp->buffer_template);
972 vlib_buffer_validate (vm, b[0]);
974 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
976 if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
978 vlib_buffer_copy_template (b[0], &bt);
979 queue[n_queue++] = bi;
982 if (n_queue == queue_size)
984 vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
988 if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
999 vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
1003 /** \brief Free buffers
1004 Frees the entire buffer chain for each buffer
1006 @param vm - (vlib_main_t *) vlib main data structure pointer
1007 @param buffers - (u32 * ) buffer index array
1008 @param n_buffers - (u32) number of buffers to free
1012 vlib_buffer_free (vlib_main_t * vm,
1013 /* pointer to first buffer */
1015 /* number of buffers to free */
1018 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
1021 /** \brief Free buffers, does not free the buffer chain for each buffer
1023 @param vm - (vlib_main_t *) vlib main data structure pointer
1024 @param buffers - (u32 * ) buffer index array
1025 @param n_buffers - (u32) number of buffers to free
1029 vlib_buffer_free_no_next (vlib_main_t * vm,
1030 /* pointer to first buffer */
1032 /* number of buffers to free */
1035 vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
1038 /** \brief Free one buffer
1039 Shorthand to free a single buffer chain.
1041 @param vm - (vlib_main_t *) vlib main data structure pointer
1042 @param buffer_index - (u32) buffer index to free
1045 vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
1047 vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
1050 /** \brief Free buffers from ring
1052 @param vm - (vlib_main_t *) vlib main data structure pointer
1053 @param buffers - (u32 * ) buffer index ring
1054 @param start - (u32) first slot in the ring
1055 @param ring_size - (u32) ring size
1056 @param n_buffers - (u32) number of buffers
1059 vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start,
1060 u32 ring_size, u32 n_buffers)
1062 ASSERT (n_buffers <= ring_size);
1064 if (PREDICT_TRUE (start + n_buffers <= ring_size))
1066 vlib_buffer_free (vm, ring + start, n_buffers);
1070 vlib_buffer_free (vm, ring + start, ring_size - start);
1071 vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
1075 /** \brief Free buffers from ring without freeing tail buffers
1077 @param vm - (vlib_main_t *) vlib main data structure pointer
1078 @param buffers - (u32 * ) buffer index ring
1079 @param start - (u32) first slot in the ring
1080 @param ring_size - (u32) ring size
1081 @param n_buffers - (u32) number of buffers
1084 vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start,
1085 u32 ring_size, u32 n_buffers)
1087 ASSERT (n_buffers <= ring_size);
1089 if (PREDICT_TRUE (start + n_buffers <= ring_size))
1091 vlib_buffer_free_no_next (vm, ring + start, n_buffers);
1095 vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
1096 vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
1100 /* Append given data to end of buffer, possibly allocating new buffers. */
1101 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
1104 /* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */
1105 #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \
1106 (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \
1107 VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL)
1109 /* duplicate all buffers in chain */
1110 always_inline vlib_buffer_t *
1111 vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
1113 vlib_buffer_t *s, *d, *fd;
1114 uword n_alloc, n_buffers = 1;
1115 u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK;
1119 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1122 s = vlib_get_buffer (vm, s->next_buffer);
1124 u32 new_buffers[n_buffers];
1126 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
1128 /* No guarantee that we'll get all the buffers we asked for */
1129 if (PREDICT_FALSE (n_alloc < n_buffers))
1132 vlib_buffer_free (vm, new_buffers, n_alloc);
1138 fd = d = vlib_get_buffer (vm, new_buffers[0]);
1139 d->current_data = s->current_data;
1140 d->current_length = s->current_length;
1141 d->flags = s->flags & flag_mask;
1142 d->trace_handle = s->trace_handle;
1143 d->total_length_not_including_first_buffer =
1144 s->total_length_not_including_first_buffer;
1145 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1146 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1147 clib_memcpy_fast (vlib_buffer_get_current (d),
1148 vlib_buffer_get_current (s), s->current_length);
1151 for (i = 1; i < n_buffers; i++)
1154 d->next_buffer = new_buffers[i];
1156 s = vlib_get_buffer (vm, s->next_buffer);
1157 d = vlib_get_buffer (vm, new_buffers[i]);
1158 d->current_data = s->current_data;
1159 d->current_length = s->current_length;
1160 clib_memcpy_fast (vlib_buffer_get_current (d),
1161 vlib_buffer_get_current (s), s->current_length);
1162 d->flags = s->flags & flag_mask;
1168 /* duplicate first buffer in chain */
1169 always_inline vlib_buffer_t *
1170 vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di)
1174 if ((vlib_buffer_alloc (vm, di, 1)) != 1)
1177 d = vlib_get_buffer (vm, *di);
1179 d->current_data = b->current_data;
1180 d->current_length = b->current_length;
1181 clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
1182 clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
1183 clib_memcpy_fast (vlib_buffer_get_current (d),
1184 vlib_buffer_get_current (b), b->current_length);
1189 /* \brief Move packet from current position to offset position in buffer.
1190 Only work for small packet using one buffer with room to fit the move
1191 @param vm - (vlib_main_t *) vlib main data structure pointer
1192 @param b - (vlib_buffer_t *) pointer to buffer
1193 @param offset - (i16) position to move the packet in buffer
1196 vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset)
1198 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1199 ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1200 ASSERT (offset + b->current_length <
1201 vlib_buffer_get_default_data_size (vm));
1203 u8 *source = vlib_buffer_get_current (b);
1204 b->current_data = offset;
1205 u8 *destination = vlib_buffer_get_current (b);
1206 u16 length = b->current_length;
1208 if (source + length <= destination) /* no overlap */
1209 clib_memcpy_fast (destination, source, length);
1211 memmove (destination, source, length);
1214 /** \brief Create a maximum of 256 clones of buffer and store them
1215 in the supplied array
1217 @param vm - (vlib_main_t *) vlib main data structure pointer
1218 @param src_buffer - (u32) source buffer index
1219 @param buffers - (u32 * ) buffer index array
1220 @param n_buffers - (u16) number of buffer clones requested (<=256)
1221 @param head_end_offset - (u16) offset relative to current position
1222 where packet head ends
1223 @param offset - (i16) copy packet head at current position if 0,
1224 else at offset position to change headroom space as specified
1225 @return - (u16) number of buffers actually cloned, may be
1226 less than the number requested or zero
1229 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1230 u16 n_buffers, u16 head_end_offset, i16 offset)
1233 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1235 ASSERT (s->ref_count == 1);
1237 ASSERT (n_buffers <= 256);
1238 ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1239 ASSERT ((offset + head_end_offset) <
1240 vlib_buffer_get_default_data_size (vm));
1242 if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
1244 buffers[0] = src_buffer;
1246 vlib_buffer_move (vm, s, offset);
1248 for (i = 1; i < n_buffers; i++)
1251 d = vlib_buffer_copy (vm, s);
1254 buffers[i] = vlib_get_buffer_index (vm, d);
1260 if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
1262 buffers[0] = src_buffer;
1266 n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
1267 s->buffer_pool_index);
1269 for (i = 0; i < n_buffers; i++)
1271 vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
1273 d->current_data = offset;
1275 d->current_data = s->current_data;
1277 d->current_length = head_end_offset;
1278 ASSERT (d->buffer_pool_index == s->buffer_pool_index);
1280 d->total_length_not_including_first_buffer = s->current_length -
1282 if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
1284 d->total_length_not_including_first_buffer +=
1285 s->total_length_not_including_first_buffer;
1287 d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) |
1288 VLIB_BUFFER_NEXT_PRESENT;
1289 d->trace_handle = s->trace_handle;
1290 clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1291 clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1292 clib_memcpy_fast (vlib_buffer_get_current (d),
1293 vlib_buffer_get_current (s), head_end_offset);
1294 d->next_buffer = src_buffer;
1296 vlib_buffer_advance (s, head_end_offset);
1297 s->ref_count = n_buffers ? n_buffers : s->ref_count;
1298 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1300 s = vlib_get_buffer (vm, s->next_buffer);
1301 s->ref_count = n_buffers ? n_buffers : s->ref_count;
1307 /** \brief Create multiple clones of buffer and store them
1308 in the supplied array
1310 @param vm - (vlib_main_t *) vlib main data structure pointer
1311 @param src_buffer - (u32) source buffer index
1312 @param buffers - (u32 * ) buffer index array
1313 @param n_buffers - (u16) number of buffer clones requested (<=256)
1314 @param head_end_offset - (u16) offset relative to current position
1315 where packet head ends
1316 @param offset - (i16) copy packet head at current position if 0,
1317 else at offset position to change headroom space as specified
1318 @return - (u16) number of buffers actually cloned, may be
1319 less than the number requested or zero
1322 vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1323 u16 n_buffers, u16 head_end_offset, i16 offset)
1325 vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1328 while (n_buffers > 256)
1330 vlib_buffer_t *copy;
1331 copy = vlib_buffer_copy (vm, s);
1332 n_cloned += vlib_buffer_clone_256 (vm,
1333 vlib_get_buffer_index (vm, copy),
1334 (buffers + n_cloned),
1335 256, head_end_offset, offset);
1338 n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
1340 n_buffers, head_end_offset, offset);
1345 /** \brief Create multiple clones of buffer and store them
1346 in the supplied array
1348 @param vm - (vlib_main_t *) vlib main data structure pointer
1349 @param src_buffer - (u32) source buffer index
1350 @param buffers - (u32 * ) buffer index array
1351 @param n_buffers - (u16) number of buffer clones requested (<=256)
1352 @param head_end_offset - (u16) offset relative to current position
1353 where packet head ends
1354 @return - (u16) number of buffers actually cloned, may be
1355 less than the number requested or zero
1358 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1359 u16 n_buffers, u16 head_end_offset)
1361 return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
1362 head_end_offset, 0);
1365 /** \brief Attach cloned tail to the buffer
1367 @param vm - (vlib_main_t *) vlib main data structure pointer
1368 @param head - (vlib_buffer_t *) head buffer
1369 @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
1373 vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
1374 vlib_buffer_t * tail)
1376 ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1377 ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
1379 head->flags |= VLIB_BUFFER_NEXT_PRESENT;
1380 head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1381 head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1382 head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1383 head->next_buffer = vlib_get_buffer_index (vm, tail);
1384 head->total_length_not_including_first_buffer = tail->current_length +
1385 tail->total_length_not_including_first_buffer;
1388 clib_atomic_add_fetch (&tail->ref_count, 1);
1390 if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
1392 tail = vlib_get_buffer (vm, tail->next_buffer);
1397 /* Initializes the buffer as an empty packet with no chained buffers. */
1399 vlib_buffer_chain_init (vlib_buffer_t * first)
1401 first->total_length_not_including_first_buffer = 0;
1402 first->current_length = 0;
1403 first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1404 first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1407 /* The provided next_bi buffer index is appended to the end of the packet. */
1408 always_inline vlib_buffer_t *
1409 vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
1411 vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
1412 last->next_buffer = next_bi;
1413 last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1414 next_buffer->current_length = 0;
1415 next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1419 /* Increases or decreases the packet length.
1420 * It does not allocate or deallocate new buffers.
1421 * Therefore, the added length must be compatible
1422 * with the last buffer. */
1424 vlib_buffer_chain_increase_length (vlib_buffer_t * first,
1425 vlib_buffer_t * last, i32 len)
1427 last->current_length += len;
1429 first->total_length_not_including_first_buffer += len;
1432 /* Copy data to the end of the packet and increases its length.
1433 * It does not allocate new buffers.
1434 * Returns the number of copied bytes. */
1436 vlib_buffer_chain_append_data (vlib_main_t * vm,
1437 vlib_buffer_t * first,
1438 vlib_buffer_t * last, void *data, u16 data_len)
1440 u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
1441 ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
1442 u16 len = clib_min (data_len,
1443 n_buffer_bytes - last->current_length -
1444 last->current_data);
1445 clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
1447 vlib_buffer_chain_increase_length (first, last, len);
1451 /* Copy data to the end of the packet and increases its length.
1452 * Allocates additional buffers from the free list if necessary.
1453 * Returns the number of copied bytes.
1454 * 'last' value is modified whenever new buffers are allocated and
1455 * chained and points to the last buffer in the chain. */
1457 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
1458 vlib_buffer_t * first,
1459 vlib_buffer_t ** last, void *data,
1461 void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);
1463 format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
1464 format_vlib_buffer_contents, format_vlib_buffer_no_chain;
1468 /* Vector of packet data. */
1471 /* Number of buffers to allocate in each call to allocator. */
1472 u32 min_n_buffers_each_alloc;
1475 } vlib_packet_template_t;
1477 void vlib_packet_template_init (vlib_main_t * vm,
1478 vlib_packet_template_t * t,
1480 uword n_packet_data_bytes,
1481 uword min_n_buffers_each_alloc,
1484 void *vlib_packet_template_get_packet (vlib_main_t * vm,
1485 vlib_packet_template_t * t,
1489 vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
1491 vec_free (t->packet_data);
1495 vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b)
1497 return b->data + vlib_buffer_get_default_data_size (vm) -
1498 ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1502 vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b)
1504 vlib_buffer_t *db = b, *sb, *first = b;
1506 u32 bytes_left = 0, data_size;
1507 u16 src_left, dst_left, n_buffers = 1;
1511 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1514 data_size = vlib_buffer_get_default_data_size (vm);
1516 dst_left = vlib_buffer_space_left_at_end (vm, b);
1518 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1520 b = vlib_get_buffer (vm, b->next_buffer);
1521 if (b->ref_count > 1)
1523 bytes_left += b->current_length;
1527 /* if buffer is cloned, create completely new chain - unless everything fits
1528 * into one buffer */
1529 if (is_cloned && bytes_left >= dst_left)
1532 u32 space_needed = bytes_left - dst_left;
1535 if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1540 b = vlib_get_buffer (vm, tail);
1542 while (len < space_needed)
1545 if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1547 vlib_buffer_free_one (vm, tail);
1550 b->flags = VLIB_BUFFER_NEXT_PRESENT;
1551 b->next_buffer = bi;
1552 b = vlib_get_buffer (vm, bi);
1556 sb = vlib_get_buffer (vm, first->next_buffer);
1557 to_free = first->next_buffer;
1558 first->next_buffer = tail;
1561 sb = vlib_get_buffer (vm, first->next_buffer);
1563 src_left = sb->current_length;
1564 sp = vlib_buffer_get_current (sb);
1565 dp = vlib_buffer_get_tail (db);
1573 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1574 ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1575 db = vlib_get_buffer (vm, db->next_buffer);
1576 dst_left = data_size;
1577 if (db->current_data > 0)
1579 db->current_data = 0;
1583 dst_left += -db->current_data;
1585 dp = vlib_buffer_get_current (db);
1588 while (src_left == 0)
1590 ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1591 sb = vlib_get_buffer (vm, sb->next_buffer);
1592 src_left = sb->current_length;
1593 sp = vlib_buffer_get_current (sb);
1596 bytes_to_copy = clib_min (dst_left, src_left);
1601 bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1603 clib_memcpy_fast (dp, sp, bytes_to_copy);
1606 src_left -= bytes_to_copy;
1607 dst_left -= bytes_to_copy;
1608 dp += bytes_to_copy;
1609 sp += bytes_to_copy;
1610 bytes_left -= bytes_to_copy;
1613 db->current_data = 0;
1614 db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1616 if (is_cloned && to_free)
1617 vlib_buffer_free_one (vm, to_free);
1620 if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1621 vlib_buffer_free_one (vm, db->next_buffer);
1622 db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1625 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1627 b = vlib_get_buffer (vm, b->next_buffer);
1632 first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1637 #endif /* included_vlib_buffer_funcs_h */
1640 * fd.io coding-style-patch-verification: ON
1643 * eval: (c-set-style "gnu")