2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer.h: VLIB buffers
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_DATA_SIZE (2048)
52 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
54 typedef u8 vlib_buffer_free_list_index_t;
57 vlib buffer structure definition and a few select
58 access methods. This structure and the buffer allocation
59 mechanism should perhaps live in vnet, but it would take a lot
60 of typing to make it so.
66 #define foreach_vlib_buffer_flag \
67 _( 0, NON_DEFAULT_FREELIST, "non-default-fl") \
69 _( 2, NEXT_PRESENT, 0) \
70 _( 3, IS_RECYCLED, "is-recycled") \
71 _( 4, TOTAL_LENGTH_VALID, 0) \
72 _( 5, REPL_FAIL, "repl-fail") \
73 _( 6, RECYCLE, "recycle") \
74 _( 7, EXT_HDR_VALID, "ext-hdr-valid")
76 /* NOTE: only buffer generic flags should be defined here, please consider
77 using user flags. i.e. src/vnet/buffer.h */
81 #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
82 foreach_vlib_buffer_flag
88 #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
89 foreach_vlib_buffer_flag
93 /* User defined buffer flags. */
94 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
95 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
97 /* VLIB buffer representation. */
100 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
101 STRUCT_MARK (template_start);
102 /* Offset within data[] that we are currently processing.
103 If negative current header points into predata area. */
104 i16 current_data; /**< signed offset in data[], pre_data[]
105 that we are currently processing.
106 If negative current header points into predata area.
108 u16 current_length; /**< Nbytes between current data and
109 the end of this buffer.
111 u32 flags; /**< buffer flags:
112 <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
113 <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
114 <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
115 <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
116 <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
117 <br> VLIB_BUFFER_RECYCLE: as it says
118 <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
119 set to avoid adding it to a flow report
120 <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
124 STRUCT_MARK (template_end);
126 u32 next_buffer; /**< Next buffer for this linked-list of buffers.
127 Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
130 vlib_error_t error; /**< Error code for buffers to be enqueued
133 u32 current_config_index; /**< Used by feature subgraph arcs to
134 visit enabled feature nodes
137 u8 feature_arc_index; /**< Used to identify feature arcs by intermediate
141 u8 n_add_refs; /**< Number of additional references to this buffer. */
143 u8 buffer_pool_index; /**< index of buffer pool this buffer belongs. */
144 u8 dont_waste_me[1]; /**< Available space in the (precious)
145 first 32 octets of buffer metadata
146 Before allocating any of it, discussion required!
149 u32 opaque[10]; /**< Opaque data used by sub-graphs for their own purposes.
150 See .../vnet/vnet/buffer.h
152 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
154 u32 trace_index; /**< Specifies index into trace buffer
155 if VLIB_PACKET_IS_TRACED flag is set.
157 u32 recycle_count; /**< Used by L2 path recycle code */
159 u32 total_length_not_including_first_buffer;
160 /**< Only valid for first buffer in chain. Current length plus
161 total length given here give total number of bytes in buffer chain.
163 vlib_buffer_free_list_index_t free_list_index; /** < only used if
164 VLIB_BUFFER_NON_DEFAULT_FREELIST
166 u8 align_pad[3]; /**< available */
167 u32 opaque2[12]; /**< More opaque data, see ../vnet/vnet/buffer.h */
169 /***** end of second cache line */
170 CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
171 u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
173 Packet rewrite string will be
174 rewritten backwards and may extend
175 back before buffer->data[0].
176 Must come directly before packet data.
179 u8 data[0]; /**< Packet data. Hardware DMA here */
180 } vlib_buffer_t; /* Must be a multiple of 64B. */
182 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
184 /** \brief Prefetch buffer metadata.
185 The first 64 bytes of buffer contains most header information
187 @param b - (vlib_buffer_t *) pointer to the buffer
188 @param type - LOAD, STORE. In most cases, STORE is the right answer
191 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
193 always_inline vlib_buffer_t *
194 vlib_buffer_next_contiguous (vlib_buffer_t * b, u32 buffer_bytes)
196 return (void *) (b + 1) + buffer_bytes;
200 vlib_buffer_struct_is_sane (vlib_buffer_t * b)
202 ASSERT (sizeof (b[0]) % 64 == 0);
204 /* Rewrite data must be before and contiguous with packet data. */
205 ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data);
208 /** \brief Get pointer to current data to process
210 @param b - (vlib_buffer_t *) pointer to the buffer
211 @return - (void *) (b->data + b->current_data)
215 vlib_buffer_get_current (vlib_buffer_t * b)
218 ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
219 return b->data + b->current_data;
222 /** \brief Advance current data pointer by the supplied (signed!) amount
224 @param b - (vlib_buffer_t *) pointer to the buffer
225 @param l - (word) signed increment
228 vlib_buffer_advance (vlib_buffer_t * b, word l)
230 ASSERT (b->current_length >= l);
231 b->current_data += l;
232 b->current_length -= l;
235 /** \brief Check if there is enough space in buffer to advance
237 @param b - (vlib_buffer_t *) pointer to the buffer
238 @param l - (word) size to check
239 @return - 0 if there is less space than 'l' in buffer
242 vlib_buffer_has_space (vlib_buffer_t * b, word l)
244 return b->current_length >= l;
247 /** \brief Reset current header & length to state they were in when
250 @param b - (vlib_buffer_t *) pointer to the buffer
254 vlib_buffer_reset (vlib_buffer_t * b)
256 b->current_length += clib_max (b->current_data, 0);
260 /** \brief Get pointer to buffer's opaque data array
262 @param b - (vlib_buffer_t *) pointer to the buffer
263 @return - (void *) b->opaque
266 vlib_get_buffer_opaque (vlib_buffer_t * b)
268 return (void *) b->opaque;
271 /** \brief Get pointer to buffer's opaque2 data array
273 @param b - (vlib_buffer_t *) pointer to the buffer
274 @return - (void *) b->opaque2
277 vlib_get_buffer_opaque2 (vlib_buffer_t * b)
279 return (void *) b->opaque2;
282 /** \brief Get pointer to the end of buffer's data
283 * @param b pointer to the buffer
284 * @return pointer to tail of packet's data
287 vlib_buffer_get_tail (vlib_buffer_t * b)
289 return b->data + b->current_data + b->current_length;
292 /** \brief Append uninitialized data to buffer
293 * @param b pointer to the buffer
294 * @param size number of uninitialized bytes
295 * @return pointer to beginning of uninitialized data
298 vlib_buffer_put_uninit (vlib_buffer_t * b, u8 size)
300 void *p = vlib_buffer_get_tail (b);
301 /* XXX make sure there's enough space */
302 b->current_length += size;
306 /** \brief Prepend uninitialized data to buffer
307 * @param b pointer to the buffer
308 * @param size number of uninitialized bytes
309 * @return pointer to beginning of uninitialized data
312 vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size)
314 ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
315 b->current_data -= size;
316 b->current_length += size;
318 return vlib_buffer_get_current (b);
321 /** \brief Make head room, typically for packet headers
322 * @param b pointer to the buffer
323 * @param size number of head room bytes
324 * @return pointer to start of buffer (current data)
327 vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
329 ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
330 b->current_data += size;
331 return vlib_buffer_get_current (b);
334 /** \brief Retrieve bytes from buffer head
335 * @param b pointer to the buffer
336 * @param size number of bytes to pull
337 * @return pointer to start of buffer (current data)
340 vlib_buffer_pull (vlib_buffer_t * b, u8 size)
342 if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size)
345 void *data = vlib_buffer_get_current (b);
346 vlib_buffer_advance (b, size);
350 /* Forward declaration. */
353 typedef struct vlib_buffer_free_list_t
355 /* Template buffer used to initialize first 16 bytes of buffers
356 allocated on this free list. */
357 vlib_buffer_t buffer_init_template;
359 /* Our index into vlib_main_t's buffer_free_list_pool. */
360 vlib_buffer_free_list_index_t index;
362 /* Number of data bytes for buffers in this free list. */
365 /* Number of buffers to allocate when we need to allocate new buffers
366 from physmem heap. */
367 u32 min_n_buffers_each_physmem_alloc;
369 /* Total number of buffers allocated from this free list. */
372 /* Vector of free buffers. Each element is a byte offset into I/O heap. */
375 /* global vector of free buffers, used only on main thread.
376 Bufers are returned to global buffers only in case when number of
377 buffers on free buffers list grows about threshold */
379 clib_spinlock_t global_buffers_lock;
381 /* Memory chunks allocated for this free list
382 recorded here so they can be freed when free list
384 void **buffer_memory_allocated;
386 /* Free list name. */
389 /* Callback functions to initialize newly allocated buffers.
390 If null buffers are zeroed. */
391 void (*buffer_init_function) (struct vlib_main_t * vm,
392 struct vlib_buffer_free_list_t * fl,
393 u32 * buffers, u32 n_buffers);
395 /* Callback function to announce that buffers have been
396 added to the freelist */
397 void (*buffers_added_to_freelist_function)
398 (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
400 uword buffer_init_function_opaque;
401 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
403 typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm,
404 vlib_buffer_free_list_t * fl,
405 uword min_free_buffers);
406 typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers,
408 typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
409 u32 * buffers, u32 n_buffers);
413 vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb;
414 vlib_buffer_free_cb_t *vlib_buffer_free_cb;
415 vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb;
416 void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
418 uword n_packet_data_bytes,
420 min_n_buffers_each_physmem_alloc,
422 void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm,
423 vlib_buffer_free_list_index_t
425 } vlib_buffer_callbacks_t;
427 extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
431 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
434 vlib_physmem_region_index_t physmem_region;
435 int alloc_chunk_size;
436 } vlib_buffer_pool_t;
440 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
441 /* Virtual memory address and size of buffer memory, used for calculating
443 uword buffer_mem_start;
444 uword buffer_mem_size;
445 vlib_buffer_pool_t *buffer_pools;
447 /* Buffer free callback, for subversive activities */
448 u32 (*buffer_free_callback) (struct vlib_main_t * vm,
450 u32 n_buffers, u32 follow_buffer_next);
451 /* Pool of buffer free lists.
452 Multiple free lists exist for packet generator which uses
453 separate free lists for each packet stream --- so as to avoid
454 initializing static data for each packet generated. */
455 vlib_buffer_free_list_t *buffer_free_list_pool;
456 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
457 #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
459 /* Hash table mapping buffer size (rounded to next unit of
460 sizeof (vlib_buffer_t)) to free list index. */
461 uword *free_list_by_size;
463 /* Hash table mapping buffer index into number
464 0 => allocated but free, 1 => allocated and not-free.
465 If buffer index is not in hash table then this buffer
466 has never been allocated. */
467 uword *buffer_known_hash;
468 clib_spinlock_t buffer_known_hash_lockp;
470 /* List of free-lists needing Blue Light Special announcements */
471 vlib_buffer_free_list_t **announce_list;
474 vlib_buffer_callbacks_t cb;
475 int callbacks_registered;
476 } vlib_buffer_main_t;
478 u8 vlib_buffer_add_physmem_region (struct vlib_main_t *vm,
479 vlib_physmem_region_index_t region);
481 clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
485 struct vlib_main_t *vlib_main;
487 u32 first_buffer, last_buffer;
493 /* Total accumulated bytes in chain starting with first_buffer. */
494 u32 n_total_data_bytes;
496 /* Max number of bytes to accumulate in chain starting with first_buffer.
497 As this limit is reached buffers are enqueued to next node. */
498 u32 max_n_data_bytes_per_chain;
500 /* Next node to enqueue buffers to relative to current process node. */
503 /* Free list to use to allocate new buffers. */
504 vlib_buffer_free_list_index_t free_list_index;
509 /* CLIB fifo of buffer indices waiting to be unserialized. */
512 /* Event type used to signal that RX buffers have been added to fifo. */
513 uword ready_one_time_event;
516 } vlib_serialize_buffer_main_t;
518 void serialize_open_vlib_buffer (serialize_main_t * m, struct vlib_main_t *vm,
519 vlib_serialize_buffer_main_t * sm);
520 void unserialize_open_vlib_buffer (serialize_main_t * m,
521 struct vlib_main_t *vm,
522 vlib_serialize_buffer_main_t * sm);
524 u32 serialize_close_vlib_buffer (serialize_main_t * m);
525 void unserialize_close_vlib_buffer (serialize_main_t * m);
526 void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
529 serialize_vlib_buffer_n_bytes (serialize_main_t * m)
531 serialize_stream_t *s = &m->stream;
532 vlib_serialize_buffer_main_t *sm
533 = uword_to_pointer (m->stream.data_function_opaque,
534 vlib_serialize_buffer_main_t *);
535 return sm->tx.n_total_data_bytes + s->current_buffer_index +
536 vec_len (s->overflow_buffer);
542 /** \brief Compile time buffer trajectory tracing option
543 Turn this on if you run into "bad monkey" contexts,
544 and you want to know exactly which nodes they've visited...
547 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
549 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
550 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
551 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
552 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
553 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
554 vlib_buffer_trace_trajectory_init (b);
556 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
557 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
559 #endif /* included_vlib_buffer_h */
561 #define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \
562 __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \
563 static void __vlib_add_buffer_callbacks_t_##x (void) \
564 __attribute__((__constructor__)) ; \
565 static void __vlib_add_buffer_callbacks_t_##x (void) \
567 if (vlib_buffer_callbacks) \
568 clib_panic ("vlib buffer callbacks already registered"); \
569 vlib_buffer_callbacks = &__##x##_buffer_callbacks; \
571 __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
574 * fd.io coding-style-patch-verification: ON
577 * eval: (c-set-style "gnu")