2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_funcs.h: VLIB buffer related functions/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
43 #include <vppinfra/hash.h>
46 vlib buffer access methods.
50 /** \brief Translate buffer index into buffer pointer
52 @param vm - (vlib_main_t *) vlib main data structure pointer
53 @param buffer_index - (u32) buffer index
54 @return - (vlib_buffer_t *) buffer pointer
56 always_inline vlib_buffer_t *
57 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
59 return vlib_physmem_at_offset (&vm->physmem_main, ((uword)buffer_index)
60 << CLIB_LOG2_CACHE_LINE_BYTES);
63 /** \brief Translate buffer pointer into buffer index
65 @param vm - (vlib_main_t *) vlib main data structure pointer
66 @param b - (void *) buffer pointer
67 @return - (u32) buffer index
70 vlib_get_buffer_index (vlib_main_t * vm, void * p)
72 uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
73 ASSERT((offset % (1<<CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
74 return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
77 /** \brief Get next buffer in buffer linklist, or zero for end of list.
79 @param vm - (vlib_main_t *) vlib main data structure pointer
80 @param b - (void *) buffer pointer
81 @return - (vlib_buffer_t *) next buffer, or NULL
83 always_inline vlib_buffer_t *
84 vlib_get_next_buffer (vlib_main_t * vm, vlib_buffer_t * b)
86 return (b->flags & VLIB_BUFFER_NEXT_PRESENT
87 ? vlib_get_buffer (vm, b->next_buffer)
91 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first);
93 /** \brief Get length in bytes of the buffer chain
95 @param vm - (vlib_main_t *) vlib main data structure pointer
96 @param b - (void *) buffer pointer
97 @return - (uword) length of buffer chain
100 vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b)
102 uword l = b->current_length + b->total_length_not_including_first_buffer;
103 if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT
104 | VLIB_BUFFER_TOTAL_LENGTH_VALID))
105 == VLIB_BUFFER_NEXT_PRESENT))
106 return vlib_buffer_length_in_chain_slow_path (vm, b);
110 /** \brief Get length in bytes of the buffer index buffer chain
112 @param vm - (vlib_main_t *) vlib main data structure pointer
113 @param bi - (u32) buffer index
114 @return - (uword) length of buffer chain
117 vlib_buffer_index_length_in_chain (vlib_main_t * vm, u32 bi)
119 vlib_buffer_t * b = vlib_get_buffer (vm, bi);
120 return vlib_buffer_length_in_chain (vm, b);
123 /** \brief Copy buffer contents to memory
125 @param vm - (vlib_main_t *) vlib main data structure pointer
126 @param bi - (u32) buffer index
127 @param contents - (u8 *) memory, <strong>must be large enough</strong>
128 @return - (uword) length of buffer chain
131 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
133 uword content_len = 0;
139 b = vlib_get_buffer (vm, buffer_index);
140 l = b->current_length;
141 memcpy (contents + content_len, b->data + b->current_data, l);
143 if (! (b->flags & VLIB_BUFFER_NEXT_PRESENT))
145 buffer_index = b->next_buffer;
151 /* Return physical address of buffer->data start. */
153 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
155 return vlib_physmem_offset_to_physical (&vm->physmem_main,
156 (((uword)buffer_index) <<
157 CLIB_LOG2_CACHE_LINE_BYTES) +
158 STRUCT_OFFSET_OF (vlib_buffer_t, data));
161 /** \brief Prefetch buffer metadata by buffer index
162 The first 64 bytes of buffer contains most header information
164 @param vm - (vlib_main_t *) vlib main data structure pointer
165 @param bi - (u32) buffer index
166 @param type - LOAD, STORE. In most cases, STORE is the right answer
168 /* Prefetch buffer header given index. */
169 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
171 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
172 vlib_prefetch_buffer_header (_b, type); \
176 /* Iterate over known allocated vlib bufs. You probably do not want
178 @param vm the vlib_main_t
179 @param bi found allocated buffer index
180 @param body operation to perform on buffer index
181 function executes body for each allocated buffer index
183 #define vlib_buffer_foreach_allocated(vm,bi,body) \
185 vlib_main_t * _vmain = (vm); \
186 vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
187 hash_pair_t * _vbpair; \
188 hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
189 if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
190 (bi) = _vbpair->key; \
200 /* Index is unknown. */
203 /* Index is known and free/allocated. */
204 VLIB_BUFFER_KNOWN_FREE,
205 VLIB_BUFFER_KNOWN_ALLOCATED,
206 } vlib_buffer_known_state_t;
208 always_inline vlib_buffer_known_state_t
209 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
211 vlib_buffer_main_t * bm = vm->buffer_main;
212 ASSERT(os_get_cpu_number() == 0);
214 uword * p = hash_get (bm->buffer_known_hash, buffer_index);
215 return p ? p[0] : VLIB_BUFFER_UNKNOWN;
219 vlib_buffer_set_known_state (vlib_main_t * vm,
221 vlib_buffer_known_state_t state)
223 vlib_buffer_main_t * bm = vm->buffer_main;
224 ASSERT(os_get_cpu_number() == 0);
225 hash_set (bm->buffer_known_hash, buffer_index, state);
228 /* Validates sanity of a single buffer.
229 Returns format'ed vector with error message if any. */
230 u8 * vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain);
232 /* Validate an array of buffers. As above. */
233 u8 * vlib_validate_buffers (vlib_main_t * vm,
235 uword next_buffer_stride,
237 vlib_buffer_known_state_t known_state,
240 #endif /* DPDK == 0 */
243 vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
244 unsigned mbuf_size, unsigned socket_id);
246 /** \brief Allocate buffers into supplied array
248 @param vm - (vlib_main_t *) vlib main data structure pointer
249 @param buffers - (u32 * ) buffer index array
250 @param n_buffers - (u32) number of buffers requested
251 @return - (u32) number of buffers actually allocated, may be
252 less than the number requested or zero
254 u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers);
257 vlib_buffer_round_size (u32 size)
258 { return round_pow2 (size, sizeof (vlib_buffer_t)); }
260 /** \brief Allocate buffers from specific freelist into supplied array
262 @param vm - (vlib_main_t *) vlib main data structure pointer
263 @param buffers - (u32 * ) buffer index array
264 @param n_buffers - (u32) number of buffers requested
265 @return - (u32) number of buffers actually allocated, may be
266 less than the number requested or zero
268 u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
271 u32 free_list_index);
273 /** \brief Free buffers
274 Frees the entire buffer chain for each buffer
276 @param vm - (vlib_main_t *) vlib main data structure pointer
277 @param buffers - (u32 * ) buffer index array
278 @param n_buffers - (u32) number of buffers to free
281 void vlib_buffer_free (vlib_main_t * vm,
282 /* pointer to first buffer */
284 /* number of buffers to free */
287 /** \brief Free buffers, does not free the buffer chain for each buffer
289 @param vm - (vlib_main_t *) vlib main data structure pointer
290 @param buffers - (u32 * ) buffer index array
291 @param n_buffers - (u32) number of buffers to free
294 void vlib_buffer_free_no_next (vlib_main_t * vm,
295 /* pointer to first buffer */
297 /* number of buffers to free */
300 /** \brief Free one buffer
301 Shorthand to free a single buffer chain.
303 @param vm - (vlib_main_t *) vlib main data structure pointer
304 @param buffer_index - (u32) buffer index to free
307 vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index)
309 vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
312 /* Add/delete buffer free lists. */
313 u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, char * fmt, ...);
314 void vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index);
316 /* Find already existing public free list with given size or create one. */
317 u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, char * fmt, ...);
319 always_inline vlib_buffer_free_list_t *
320 vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
322 vlib_buffer_main_t * bm = vm->buffer_main;
323 vlib_buffer_free_list_t * f;
325 f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
327 /* Sanity: indices must match. */
328 ASSERT (f->index == free_list_index);
334 vlib_buffer_free_list_buffer_size (vlib_main_t * vm, u32 free_list_index)
336 vlib_buffer_free_list_t * f = vlib_buffer_get_free_list (vm, free_list_index);
337 return f->n_data_bytes;
341 vlib_aligned_memcpy (void * _dst, void * _src, int n_bytes);
343 /* Reasonably fast buffer copy routine. */
345 vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
367 vlib_physmem_alloc_aligned (vlib_main_t * vm, clib_error_t ** error,
368 uword n_bytes, uword alignment)
370 void * r = vm->os_physmem_alloc_aligned (&vm->physmem_main, n_bytes, alignment);
372 *error = clib_error_return (0, "failed to allocate %wd bytes of I/O memory", n_bytes);
378 /* By default allocate I/O memory with cache line alignment. */
380 vlib_physmem_alloc (vlib_main_t * vm, clib_error_t ** error, uword n_bytes)
381 { return vlib_physmem_alloc_aligned (vm, error, n_bytes, CLIB_CACHE_LINE_BYTES); }
384 vlib_physmem_free (vlib_main_t * vm, void * mem)
385 { return vm->os_physmem_free (mem); }
388 vlib_physmem_virtual_to_physical (vlib_main_t * vm, void * mem)
390 vlib_physmem_main_t * pm = &vm->physmem_main;
391 uword o = pointer_to_uword (mem) - pm->virtual.start;
392 return vlib_physmem_offset_to_physical (pm, o);
395 /* Append given data to end of buffer, possibly allocating new buffers. */
396 u32 vlib_buffer_add_data (vlib_main_t * vm,
399 void * data, u32 n_data_bytes);
401 format_function_t format_vlib_buffer, format_vlib_buffer_and_data, format_vlib_buffer_contents;
404 /* Vector of packet data. */
408 /* Number of buffers to allocate in each call to physmem
410 u32 min_n_buffers_each_physmem_alloc;
412 /* Buffer free list for this template. */
417 } vlib_packet_template_t;
419 void vlib_packet_template_get_packet_helper (vlib_main_t * vm,
420 vlib_packet_template_t * t);
422 void vlib_packet_template_init (vlib_main_t * vm,
423 vlib_packet_template_t * t,
425 uword n_packet_data_bytes,
426 uword min_n_buffers_each_physmem_alloc,
430 vlib_packet_template_get_packet (vlib_main_t * vm,
431 vlib_packet_template_t * t,
435 vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t)
437 vec_free (t->packet_data);
441 unserialize_vlib_buffer_n_bytes (serialize_main_t * m)
443 serialize_stream_t * s = &m->stream;
444 vlib_serialize_buffer_main_t * sm
445 = uword_to_pointer (m->stream.data_function_opaque, vlib_serialize_buffer_main_t *);
446 vlib_main_t * vm = sm->vlib_main;
449 n = s->n_buffer_bytes - s->current_buffer_index;
450 if (sm->last_buffer != ~0)
452 vlib_buffer_t * b = vlib_get_buffer (vm, sm->last_buffer);
453 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
455 b = vlib_get_buffer (vm, b->next_buffer);
456 n += b->current_length;
460 clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
461 n += vlib_buffer_index_length_in_chain (vm, f[0]);
469 vlib_copy_unit_t i[sizeof (vlib_buffer_t) / sizeof (vlib_copy_unit_t)];
470 } vlib_buffer_union_t;
472 /* Set a buffer quickly into "uninitialized" state. We want this to
473 be extremely cheap and arrange for all fields that need to be
474 initialized to be in the first 128 bits of the buffer. */
476 vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
477 vlib_buffer_free_list_t * fl)
479 vlib_buffer_union_t * dst = (vlib_buffer_union_t *) _dst;
480 vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template;
482 /* Make sure buffer template is sane. */
483 ASSERT (fl->index == fl->buffer_init_template.free_list_index);
485 /* Copy template from src->current_data thru src->free_list_index */
486 dst->i[0] = src->i[0];
487 if (1 * sizeof (dst->i[0]) < 16)
488 dst->i[1] = src->i[1];
489 if (2 * sizeof (dst->i[0]) < 16)
490 dst->i[2] = src->i[2];
492 /* Make sure it really worked. */
493 #define _(f) ASSERT (dst->b.f == src->b.f)
499 ASSERT (dst->b.total_length_not_including_first_buffer == 0);
503 vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
504 vlib_buffer_t * _dst1,
505 vlib_buffer_free_list_t * fl)
507 vlib_buffer_union_t * dst0 = (vlib_buffer_union_t *) _dst0;
508 vlib_buffer_union_t * dst1 = (vlib_buffer_union_t *) _dst1;
509 vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template;
511 /* Make sure buffer template is sane. */
512 ASSERT (fl->index == fl->buffer_init_template.free_list_index);
514 /* Copy template from src->current_data thru src->free_list_index */
515 dst0->i[0] = dst1->i[0] = src->i[0];
516 if (1 * sizeof (dst0->i[0]) < 16)
517 dst0->i[1] = dst1->i[1] = src->i[1];
518 if (2 * sizeof (dst0->i[0]) < 16)
519 dst0->i[2] = dst1->i[2] = src->i[2];
521 /* Make sure it really worked. */
522 #define _(f) ASSERT (dst0->b.f == src->b.f && dst1->b.f == src->b.f)
528 ASSERT (dst0->b.total_length_not_including_first_buffer == 0);
529 ASSERT (dst1->b.total_length_not_including_first_buffer == 0);
533 u32 * vlib_buffer_state_validation_lock;
534 uword * vlib_buffer_state_validation_hash;
535 void * vlib_buffer_state_heap;
539 vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected)
545 oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
547 while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
550 p = hash_get (vlib_buffer_state_validation_hash, b);
552 /* If we don't know about b, declare it to be in the expected state */
555 hash_set (vlib_buffer_state_validation_hash, b, expected);
559 if (p[0] != expected)
563 vlib_main_t * vm = &vlib_global_main;
567 bi = vlib_get_buffer_index (vm, b);
569 clib_mem_set_heap (oldheap);
570 clib_warning ("%.6f buffer %llx (%d): %s, not %s",
571 vlib_time_now(vm), bi,
572 p[0] ? "busy" : "free",
573 expected ? "busy" : "free");
577 CLIB_MEMORY_BARRIER();
578 *vlib_buffer_state_validation_lock = 0;
579 clib_mem_set_heap (oldheap);
584 vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected)
589 oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
591 while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
594 hash_set (vlib_buffer_state_validation_hash, b, expected);
596 CLIB_MEMORY_BARRIER();
597 *vlib_buffer_state_validation_lock = 0;
598 clib_mem_set_heap (oldheap);
602 #endif /* included_vlib_buffer_funcs_h */