2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_funcs.h: VLIB buffer related functions/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
43 #include <vppinfra/hash.h>
46 vlib buffer access methods.
50 /** \brief Translate buffer index into buffer pointer
52 @param vm - (vlib_main_t *) vlib main data structure pointer
53 @param buffer_index - (u32) buffer index
54 @return - (vlib_buffer_t *) buffer pointer
56 always_inline vlib_buffer_t *
57 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
59 return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
60 << CLIB_LOG2_CACHE_LINE_BYTES);
63 /** \brief Translate buffer pointer into buffer index
65 @param vm - (vlib_main_t *) vlib main data structure pointer
66 @param p - (void *) buffer pointer
67 @return - (u32) buffer index
70 vlib_get_buffer_index (vlib_main_t * vm, void *p)
72 uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
73 ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
74 return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
77 /** \brief Get next buffer in buffer linklist, or zero for end of list.
79 @param vm - (vlib_main_t *) vlib main data structure pointer
80 @param b - (void *) buffer pointer
81 @return - (vlib_buffer_t *) next buffer, or NULL
83 always_inline vlib_buffer_t *
84 vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
86 return (b->flags & VLIB_BUFFER_NEXT_PRESENT
87 ? vlib_get_buffer (vm, b->next_buffer) : 0);
90 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
91 vlib_buffer_t * b_first);
93 /** \brief Get length in bytes of the buffer chain
95 @param vm - (vlib_main_t *) vlib main data structure pointer
96 @param b - (void *) buffer pointer
97 @return - (uword) length of buffer chain
100 vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
102 uword l = b->current_length + b->total_length_not_including_first_buffer;
103 if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
104 | VLIB_BUFFER_TOTAL_LENGTH_VALID))
105 == VLIB_BUFFER_NEXT_PRESENT))
106 return vlib_buffer_length_in_chain_slow_path (vm, b);
110 /** \brief Get length in bytes of the buffer index buffer chain
112 @param vm - (vlib_main_t *) vlib main data structure pointer
113 @param bi - (u32) buffer index
114 @return - (uword) length of buffer chain
117 vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
119 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
120 return vlib_buffer_length_in_chain (vm, b);
123 /** \brief Copy buffer contents to memory
125 @param vm - (vlib_main_t *) vlib main data structure pointer
126 @param buffer_index - (u32) buffer index
127 @param contents - (u8 *) memory, <strong>must be large enough</strong>
128 @return - (uword) length of buffer chain
131 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
133 uword content_len = 0;
139 b = vlib_get_buffer (vm, buffer_index);
140 l = b->current_length;
141 clib_memcpy (contents + content_len, b->data + b->current_data, l);
143 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
145 buffer_index = b->next_buffer;
151 /* Return physical address of buffer->data start. */
153 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
155 return vlib_physmem_offset_to_physical (&vm->physmem_main,
156 (((uword) buffer_index) <<
157 CLIB_LOG2_CACHE_LINE_BYTES) +
158 STRUCT_OFFSET_OF (vlib_buffer_t,
162 /** \brief Prefetch buffer metadata by buffer index
163 The first 64 bytes of buffer contains most header information
165 @param vm - (vlib_main_t *) vlib main data structure pointer
166 @param bi - (u32) buffer index
167 @param type - LOAD, STORE. In most cases, STORE is the right answer
169 /* Prefetch buffer header given index. */
170 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
172 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
173 vlib_prefetch_buffer_header (_b, type); \
177 /* Iterate over known allocated vlib bufs. You probably do not want
179 @param vm the vlib_main_t
180 @param bi found allocated buffer index
181 @param body operation to perform on buffer index
182 function executes body for each allocated buffer index
184 #define vlib_buffer_foreach_allocated(vm,bi,body) \
186 vlib_main_t * _vmain = (vm); \
187 vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
188 hash_pair_t * _vbpair; \
189 hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
190 if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
191 (bi) = _vbpair->key; \
202 /* Index is unknown. */
205 /* Index is known and free/allocated. */
206 VLIB_BUFFER_KNOWN_FREE,
207 VLIB_BUFFER_KNOWN_ALLOCATED,
208 } vlib_buffer_known_state_t;
210 always_inline vlib_buffer_known_state_t
211 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
213 vlib_buffer_main_t *bm = vm->buffer_main;
214 ASSERT (os_get_cpu_number () == 0);
216 uword *p = hash_get (bm->buffer_known_hash, buffer_index);
217 return p ? p[0] : VLIB_BUFFER_UNKNOWN;
221 vlib_buffer_set_known_state (vlib_main_t * vm,
223 vlib_buffer_known_state_t state)
225 vlib_buffer_main_t *bm = vm->buffer_main;
226 ASSERT (os_get_cpu_number () == 0);
227 hash_set (bm->buffer_known_hash, buffer_index, state);
230 /* Validates sanity of a single buffer.
231 Returns format'ed vector with error message if any. */
232 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
235 #endif /* DPDK == 0 */
237 clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
240 /** \brief Allocate buffers into supplied array
242 @param vm - (vlib_main_t *) vlib main data structure pointer
243 @param buffers - (u32 * ) buffer index array
244 @param n_buffers - (u32) number of buffers requested
245 @return - (u32) number of buffers actually allocated, may be
246 less than the number requested or zero
248 u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers);
251 vlib_buffer_round_size (u32 size)
253 return round_pow2 (size, sizeof (vlib_buffer_t));
256 /** \brief Allocate buffers from specific freelist into supplied array
258 @param vm - (vlib_main_t *) vlib main data structure pointer
259 @param buffers - (u32 * ) buffer index array
260 @param n_buffers - (u32) number of buffers requested
261 @return - (u32) number of buffers actually allocated, may be
262 less than the number requested or zero
264 u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
266 u32 n_buffers, u32 free_list_index);
268 /** \brief Free buffers
269 Frees the entire buffer chain for each buffer
271 @param vm - (vlib_main_t *) vlib main data structure pointer
272 @param buffers - (u32 * ) buffer index array
273 @param n_buffers - (u32) number of buffers to free
276 void vlib_buffer_free (vlib_main_t * vm,
277 /* pointer to first buffer */
279 /* number of buffers to free */
282 /** \brief Free buffers, does not free the buffer chain for each buffer
284 @param vm - (vlib_main_t *) vlib main data structure pointer
285 @param buffers - (u32 * ) buffer index array
286 @param n_buffers - (u32) number of buffers to free
289 void vlib_buffer_free_no_next (vlib_main_t * vm,
290 /* pointer to first buffer */
292 /* number of buffers to free */
295 /** \brief Free one buffer
296 Shorthand to free a single buffer chain.
298 @param vm - (vlib_main_t *) vlib main data structure pointer
299 @param buffer_index - (u32) buffer index to free
302 vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
304 vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
307 /* Add/delete buffer free lists. */
308 u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
310 void vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index);
312 /* Find already existing public free list with given size or create one. */
313 u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
316 always_inline vlib_buffer_free_list_t *
317 vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
319 vlib_buffer_main_t *bm = vm->buffer_main;
320 vlib_buffer_free_list_t *f;
322 f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
324 /* Sanity: indices must match. */
325 ASSERT (f->index == free_list_index);
331 vlib_buffer_free_list_buffer_size (vlib_main_t * vm, u32 free_list_index)
333 vlib_buffer_free_list_t *f =
334 vlib_buffer_get_free_list (vm, free_list_index);
335 return f->n_data_bytes;
338 void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
340 /* Reasonably fast buffer copy routine. */
342 vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
364 vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
365 uword n_bytes, uword alignment)
368 vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
371 clib_error_return (0, "failed to allocate %wd bytes of I/O memory",
378 /* By default allocate I/O memory with cache line alignment. */
380 vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
382 return vlib_physmem_alloc_aligned (vm, error, n_bytes,
383 CLIB_CACHE_LINE_BYTES);
387 vlib_physmem_free (vlib_main_t * vm, void *mem)
389 return vm->os_physmem_free (mem);
393 vlib_physmem_virtual_to_physical (vlib_main_t * vm, void *mem)
395 vlib_physmem_main_t *pm = &vm->physmem_main;
396 uword o = pointer_to_uword (mem) - pm->virtual.start;
397 return vlib_physmem_offset_to_physical (pm, o);
400 /* Append given data to end of buffer, possibly allocating new buffers. */
401 u32 vlib_buffer_add_data (vlib_main_t * vm,
403 u32 buffer_index, void *data, u32 n_data_bytes);
405 /* duplicate all buffers in chain */
406 always_inline vlib_buffer_t *
407 vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
409 vlib_buffer_t *s, *d, *fd;
410 uword n_alloc, n_buffers = 1;
411 u32 *new_buffers = 0;
412 u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
416 while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
419 s = vlib_get_buffer (vm, s->next_buffer);
422 vec_validate (new_buffers, n_buffers - 1);
423 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
424 ASSERT (n_alloc == n_buffers);
428 fd = d = vlib_get_buffer (vm, new_buffers[0]);
429 d->current_data = s->current_data;
430 d->current_length = s->current_length;
431 d->flags = s->flags & flag_mask;
432 d->total_length_not_including_first_buffer =
433 s->total_length_not_including_first_buffer;
434 clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
435 clib_memcpy (vlib_buffer_get_current (d),
436 vlib_buffer_get_current (s), s->current_length);
439 for (i = 1; i < n_buffers; i++)
442 d->next_buffer = new_buffers[i];
444 s = vlib_get_buffer (vm, s->next_buffer);
445 d = vlib_get_buffer (vm, new_buffers[i]);
446 d->current_data = s->current_data;
447 d->current_length = s->current_length;
448 clib_memcpy (vlib_buffer_get_current (d),
449 vlib_buffer_get_current (s), s->current_length);
450 d->flags = s->flags & flag_mask;
457 * vlib_buffer_chain_* functions provide a way to create long buffers.
458 * When DPDK is enabled, the 'hidden' DPDK header is taken care of transparently.
461 /* Initializes the buffer as an empty packet with no chained buffers. */
463 vlib_buffer_chain_init (vlib_buffer_t * first)
465 first->total_length_not_including_first_buffer = 0;
466 first->current_length = 0;
467 first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
468 first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
471 /* The provided next_bi buffer index is appended to the end of the packet. */
472 always_inline vlib_buffer_t *
473 vlib_buffer_chain_buffer (vlib_main_t * vm,
474 vlib_buffer_t * first,
475 vlib_buffer_t * last, u32 next_bi)
477 vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
478 last->next_buffer = next_bi;
479 last->flags |= VLIB_BUFFER_NEXT_PRESENT;
480 next_buffer->current_length = 0;
481 next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
485 /* Increases or decreases the packet length.
486 * It does not allocate or deallocate new buffers.
487 * Therefore, the added length must be compatible
488 * with the last buffer. */
490 vlib_buffer_chain_increase_length (vlib_buffer_t * first,
491 vlib_buffer_t * last, i32 len)
493 last->current_length += len;
495 first->total_length_not_including_first_buffer += len;
498 /* Copy data to the end of the packet and increases its length.
499 * It does not allocate new buffers.
500 * Returns the number of copied bytes. */
502 vlib_buffer_chain_append_data (vlib_main_t * vm,
504 vlib_buffer_t * first,
505 vlib_buffer_t * last, void *data, u16 data_len)
508 vlib_buffer_free_list_buffer_size (vm, free_list_index);
509 ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
510 u16 len = clib_min (data_len,
511 n_buffer_bytes - last->current_length -
513 clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
515 vlib_buffer_chain_increase_length (first, last, len);
519 /* Copy data to the end of the packet and increases its length.
520 * Allocates additional buffers from the free list if necessary.
521 * Returns the number of copied bytes.
522 * 'last' value is modified whenever new buffers are allocated and
523 * chained and points to the last buffer in the chain. */
525 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
527 vlib_buffer_t * first,
528 vlib_buffer_t ** last,
529 void *data, u16 data_len);
530 void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first);
532 format_function_t format_vlib_buffer, format_vlib_buffer_and_data,
533 format_vlib_buffer_contents;
537 /* Vector of packet data. */
540 /* Note: the next three fields are unused if DPDK == 1 */
542 /* Number of buffers to allocate in each call to physmem
544 u32 min_n_buffers_each_physmem_alloc;
546 /* Buffer free list for this template. */
550 } vlib_packet_template_t;
552 void vlib_packet_template_get_packet_helper (vlib_main_t * vm,
553 vlib_packet_template_t * t);
555 void vlib_packet_template_init (vlib_main_t * vm,
556 vlib_packet_template_t * t,
558 uword n_packet_data_bytes,
559 uword min_n_buffers_each_physmem_alloc,
562 void *vlib_packet_template_get_packet (vlib_main_t * vm,
563 vlib_packet_template_t * t,
567 vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
569 vec_free (t->packet_data);
573 unserialize_vlib_buffer_n_bytes (serialize_main_t * m)
575 serialize_stream_t *s = &m->stream;
576 vlib_serialize_buffer_main_t *sm
577 = uword_to_pointer (m->stream.data_function_opaque,
578 vlib_serialize_buffer_main_t *);
579 vlib_main_t *vm = sm->vlib_main;
582 n = s->n_buffer_bytes - s->current_buffer_index;
583 if (sm->last_buffer != ~0)
585 vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer);
586 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
588 b = vlib_get_buffer (vm, b->next_buffer);
589 n += b->current_length;
594 clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
595 n += vlib_buffer_index_length_in_chain (vm, f[0]);
605 vlib_copy_unit_t i[sizeof (vlib_buffer_t) / sizeof (vlib_copy_unit_t)];
609 /* Set a buffer quickly into "uninitialized" state. We want this to
610 be extremely cheap and arrange for all fields that need to be
611 initialized to be in the first 128 bits of the buffer. */
613 vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
614 vlib_buffer_free_list_t * fl)
616 vlib_buffer_union_t *dst = (vlib_buffer_union_t *) _dst;
617 vlib_buffer_union_t *src =
618 (vlib_buffer_union_t *) & fl->buffer_init_template;
620 /* Make sure vlib_buffer_t is cacheline aligned and sized */
621 ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
622 ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
623 CLIB_CACHE_LINE_BYTES);
624 ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
625 CLIB_CACHE_LINE_BYTES * 2);
627 /* Make sure buffer template is sane. */
628 ASSERT (fl->index == fl->buffer_init_template.free_list_index);
630 /* Copy template from src->current_data thru src->free_list_index */
631 dst->i[0] = src->i[0];
632 if (1 * sizeof (dst->i[0]) < 16)
633 dst->i[1] = src->i[1];
634 if (2 * sizeof (dst->i[0]) < 16)
635 dst->i[2] = src->i[2];
637 /* Make sure it really worked. */
638 #define _(f) ASSERT (dst->b.f == src->b.f)
644 ASSERT (dst->b.total_length_not_including_first_buffer == 0);
648 vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
649 vlib_buffer_t * _dst1,
650 vlib_buffer_free_list_t * fl)
652 vlib_buffer_union_t *dst0 = (vlib_buffer_union_t *) _dst0;
653 vlib_buffer_union_t *dst1 = (vlib_buffer_union_t *) _dst1;
654 vlib_buffer_union_t *src =
655 (vlib_buffer_union_t *) & fl->buffer_init_template;
657 /* Make sure buffer template is sane. */
658 ASSERT (fl->index == fl->buffer_init_template.free_list_index);
660 /* Copy template from src->current_data thru src->free_list_index */
661 dst0->i[0] = dst1->i[0] = src->i[0];
662 if (1 * sizeof (dst0->i[0]) < 16)
663 dst0->i[1] = dst1->i[1] = src->i[1];
664 if (2 * sizeof (dst0->i[0]) < 16)
665 dst0->i[2] = dst1->i[2] = src->i[2];
667 /* Make sure it really worked. */
668 #define _(f) ASSERT (dst0->b.f == src->b.f && dst1->b.f == src->b.f)
674 ASSERT (dst0->b.total_length_not_including_first_buffer == 0);
675 ASSERT (dst1->b.total_length_not_including_first_buffer == 0);
679 extern u32 *vlib_buffer_state_validation_lock;
680 extern uword *vlib_buffer_state_validation_hash;
681 extern void *vlib_buffer_state_heap;
685 vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected)
691 oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
693 while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
696 p = hash_get (vlib_buffer_state_validation_hash, b);
698 /* If we don't know about b, declare it to be in the expected state */
701 hash_set (vlib_buffer_state_validation_hash, b, expected);
705 if (p[0] != expected)
709 vlib_main_t *vm = &vlib_global_main;
713 bi = vlib_get_buffer_index (vm, b);
715 clib_mem_set_heap (oldheap);
716 clib_warning ("%.6f buffer %llx (%d): %s, not %s",
717 vlib_time_now (vm), bi,
718 p[0] ? "busy" : "free", expected ? "busy" : "free");
722 CLIB_MEMORY_BARRIER ();
723 *vlib_buffer_state_validation_lock = 0;
724 clib_mem_set_heap (oldheap);
729 vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected)
734 oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
736 while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
739 hash_set (vlib_buffer_state_validation_hash, b, expected);
741 CLIB_MEMORY_BARRIER ();
742 *vlib_buffer_state_validation_lock = 0;
743 clib_mem_set_heap (oldheap);
747 #endif /* included_vlib_buffer_funcs_h */
750 * fd.io coding-style-patch-verification: ON
753 * eval: (c-set-style "gnu")