2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer.h: VLIB buffers
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_DATA_SIZE (2048)
52 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
55 vlib buffer structure definition and a few select
56 access methods. This structure and the buffer allocation
57 mechanism should perhaps live in vnet, but it would take a lot
58 of typing to make it so.
61 /* VLIB buffer representation. */
64 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
65 STRUCT_MARK (template_start);
66 /* Offset within data[] that we are currently processing.
67 If negative current header points into predata area. */
68 i16 current_data; /**< signed offset in data[], pre_data[]
69 that we are currently processing.
70 If negative current header points into predata area.
72 u16 current_length; /**< Nbytes between current data and
73 the end of this buffer.
75 u32 flags; /**< buffer flags:
76 <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
77 <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
78 <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
79 <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
80 <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
81 <br> VLIB_BUFFER_RECYCLE: as it says
82 <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report,
83 <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
84 set to avoid adding it to a flow report
85 <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
88 /* any change to the following line requres update of
89 * vlib_buffer_get_free_list_index(...) and
90 * vlib_buffer_set_free_list_index(...) functions */
91 #define VLIB_BUFFER_FREE_LIST_INDEX_MASK ((1 << 5) - 1)
93 #define VLIB_BUFFER_IS_TRACED (1 << 5)
94 #define VLIB_BUFFER_LOG2_NEXT_PRESENT (6)
95 #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT)
96 #define VLIB_BUFFER_IS_RECYCLED (1 << 7)
97 #define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 8)
98 #define VLIB_BUFFER_REPL_FAIL (1 << 9)
99 #define VLIB_BUFFER_RECYCLE (1 << 10)
100 #define VLIB_BUFFER_FLOW_REPORT (1 << 11)
101 #define VLIB_BUFFER_EXT_HDR_VALID (1 << 12)
103 /* User defined buffer flags. */
104 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
105 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
107 STRUCT_MARK (template_end);
109 u32 next_buffer; /**< Next buffer for this linked-list of buffers.
110 Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
113 vlib_error_t error; /**< Error code for buffers to be enqueued
116 u32 current_config_index; /**< Used by feature subgraph arcs to
117 visit enabled feature nodes
120 u8 feature_arc_index; /**< Used to identify feature arcs by intermediate
124 u8 n_add_refs; /**< Number of additional references to this buffer. */
126 u8 buffer_pool_index; /**< index of buffer pool this buffer belongs. */
127 u8 dont_waste_me[1]; /**< Available space in the (precious)
128 first 32 octets of buffer metadata
129 Before allocating any of it, discussion required!
132 u32 opaque[10]; /**< Opaque data used by sub-graphs for their own purposes.
133 See .../vnet/vnet/buffer.h
135 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
137 u32 trace_index; /**< Specifies index into trace buffer
138 if VLIB_PACKET_IS_TRACED flag is set.
140 u32 recycle_count; /**< Used by L2 path recycle code */
142 u32 total_length_not_including_first_buffer;
143 /**< Only valid for first buffer in chain. Current length plus
144 total length given here give total number of bytes in buffer chain.
146 u32 align_pad; /**< available */
147 u32 opaque2[12]; /**< More opaque data, see ../vnet/vnet/buffer.h */
149 /***** end of second cache line */
150 CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
151 u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
153 Packet rewrite string will be
154 rewritten backwards and may extend
155 back before buffer->data[0].
156 Must come directly before packet data.
159 u8 data[0]; /**< Packet data. Hardware DMA here */
160 } vlib_buffer_t; /* Must be a multiple of 64B. */
162 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
164 /** \brief Prefetch buffer metadata.
165 The first 64 bytes of buffer contains most header information
167 @param b - (vlib_buffer_t *) pointer to the buffer
168 @param type - LOAD, STORE. In most cases, STORE is the right answer
171 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
173 always_inline vlib_buffer_t *
174 vlib_buffer_next_contiguous (vlib_buffer_t * b, u32 buffer_bytes)
176 return (void *) (b + 1) + buffer_bytes;
180 vlib_buffer_struct_is_sane (vlib_buffer_t * b)
182 ASSERT (sizeof (b[0]) % 64 == 0);
184 /* Rewrite data must be before and contiguous with packet data. */
185 ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data);
188 /** \brief Get pointer to current data to process
190 @param b - (vlib_buffer_t *) pointer to the buffer
191 @return - (void *) (b->data + b->current_data)
195 vlib_buffer_get_current (vlib_buffer_t * b)
198 ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
199 return b->data + b->current_data;
202 /** \brief Advance current data pointer by the supplied (signed!) amount
204 @param b - (vlib_buffer_t *) pointer to the buffer
205 @param l - (word) signed increment
208 vlib_buffer_advance (vlib_buffer_t * b, word l)
210 ASSERT (b->current_length >= l);
211 b->current_data += l;
212 b->current_length -= l;
215 /** \brief Check if there is enough space in buffer to advance
217 @param b - (vlib_buffer_t *) pointer to the buffer
218 @param l - (word) size to check
219 @return - 0 if there is less space than 'l' in buffer
222 vlib_buffer_has_space (vlib_buffer_t * b, word l)
224 return b->current_length >= l;
227 /** \brief Reset current header & length to state they were in when
230 @param b - (vlib_buffer_t *) pointer to the buffer
234 vlib_buffer_reset (vlib_buffer_t * b)
236 b->current_length += clib_max (b->current_data, 0);
240 /** \brief Get pointer to buffer's opaque data array
242 @param b - (vlib_buffer_t *) pointer to the buffer
243 @return - (void *) b->opaque
246 vlib_get_buffer_opaque (vlib_buffer_t * b)
248 return (void *) b->opaque;
251 /** \brief Get pointer to buffer's opaque2 data array
253 @param b - (vlib_buffer_t *) pointer to the buffer
254 @return - (void *) b->opaque2
257 vlib_get_buffer_opaque2 (vlib_buffer_t * b)
259 return (void *) b->opaque2;
262 /** \brief Get pointer to the end of buffer's data
263 * @param b pointer to the buffer
264 * @return pointer to tail of packet's data
267 vlib_buffer_get_tail (vlib_buffer_t * b)
269 return b->data + b->current_data + b->current_length;
272 /** \brief Append uninitialized data to buffer
273 * @param b pointer to the buffer
274 * @param size number of uninitialized bytes
275 * @return pointer to beginning of uninitialized data
278 vlib_buffer_put_uninit (vlib_buffer_t * b, u8 size)
280 void *p = vlib_buffer_get_tail (b);
281 /* XXX make sure there's enough space */
282 b->current_length += size;
286 /** \brief Prepend uninitialized data to buffer
287 * @param b pointer to the buffer
288 * @param size number of uninitialized bytes
289 * @return pointer to beginning of uninitialized data
292 vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size)
294 ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
295 b->current_data -= size;
296 b->current_length += size;
298 return vlib_buffer_get_current (b);
301 /** \brief Make head room, typically for packet headers
302 * @param b pointer to the buffer
303 * @param size number of head room bytes
304 * @return pointer to start of buffer (current data)
307 vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
309 ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
310 b->current_data += size;
311 return vlib_buffer_get_current (b);
314 /** \brief Retrieve bytes from buffer head
315 * @param b pointer to the buffer
316 * @param size number of bytes to pull
317 * @return pointer to start of buffer (current data)
320 vlib_buffer_pull (vlib_buffer_t * b, u8 size)
322 if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size)
325 void *data = vlib_buffer_get_current (b);
326 vlib_buffer_advance (b, size);
330 /* Forward declaration. */
333 typedef struct vlib_buffer_free_list_t
335 /* Template buffer used to initialize first 16 bytes of buffers
336 allocated on this free list. */
337 vlib_buffer_t buffer_init_template;
339 /* Our index into vlib_main_t's buffer_free_list_pool. */
342 /* Number of data bytes for buffers in this free list. */
345 /* Number of buffers to allocate when we need to allocate new buffers
346 from physmem heap. */
347 u32 min_n_buffers_each_physmem_alloc;
349 /* Total number of buffers allocated from this free list. */
352 /* Vector of free buffers. Each element is a byte offset into I/O heap. */
355 /* global vector of free buffers, used only on main thread.
356 Bufers are returned to global buffers only in case when number of
357 buffers on free buffers list grows about threshold */
359 clib_spinlock_t global_buffers_lock;
361 /* Memory chunks allocated for this free list
362 recorded here so they can be freed when free list
364 void **buffer_memory_allocated;
366 /* Free list name. */
369 /* Callback functions to initialize newly allocated buffers.
370 If null buffers are zeroed. */
371 void (*buffer_init_function) (struct vlib_main_t * vm,
372 struct vlib_buffer_free_list_t * fl,
373 u32 * buffers, u32 n_buffers);
375 /* Callback function to announce that buffers have been
376 added to the freelist */
377 void (*buffers_added_to_freelist_function)
378 (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
380 uword buffer_init_function_opaque;
381 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
383 typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers,
385 typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm,
388 u32 free_list_index);
389 typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers,
391 typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
392 u32 * buffers, u32 n_buffers);
396 vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb;
397 vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb;
398 vlib_buffer_free_cb_t *vlib_buffer_free_cb;
399 vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb;
400 void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
402 uword n_packet_data_bytes,
404 min_n_buffers_each_physmem_alloc,
406 void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm,
407 u32 free_list_index);
408 } vlib_buffer_callbacks_t;
410 extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
414 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
417 vlib_physmem_region_index_t physmem_region;
418 } vlib_buffer_pool_t;
422 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
423 /* Virtual memory address and size of buffer memory, used for calculating
425 uword buffer_mem_start;
426 uword buffer_mem_size;
427 vlib_buffer_pool_t *buffer_pools;
429 /* Buffer free callback, for subversive activities */
430 u32 (*buffer_free_callback) (struct vlib_main_t * vm,
432 u32 n_buffers, u32 follow_buffer_next);
433 /* Pool of buffer free lists.
434 Multiple free lists exist for packet generator which uses
435 separate free lists for each packet stream --- so as to avoid
436 initializing static data for each packet generated. */
437 vlib_buffer_free_list_t *buffer_free_list_pool;
438 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
439 #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
441 /* Hash table mapping buffer size (rounded to next unit of
442 sizeof (vlib_buffer_t)) to free list index. */
443 uword *free_list_by_size;
445 /* Hash table mapping buffer index into number
446 0 => allocated but free, 1 => allocated and not-free.
447 If buffer index is not in hash table then this buffer
448 has never been allocated. */
449 uword *buffer_known_hash;
450 clib_spinlock_t buffer_known_hash_lockp;
452 /* List of free-lists needing Blue Light Special announcements */
453 vlib_buffer_free_list_t **announce_list;
456 vlib_buffer_callbacks_t cb;
457 int callbacks_registered;
458 } vlib_buffer_main_t;
460 u8 vlib_buffer_add_physmem_region (struct vlib_main_t *vm,
461 vlib_physmem_region_index_t region);
463 clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
467 struct vlib_main_t *vlib_main;
469 u32 first_buffer, last_buffer;
475 /* Total accumulated bytes in chain starting with first_buffer. */
476 u32 n_total_data_bytes;
478 /* Max number of bytes to accumulate in chain starting with first_buffer.
479 As this limit is reached buffers are enqueued to next node. */
480 u32 max_n_data_bytes_per_chain;
482 /* Next node to enqueue buffers to relative to current process node. */
485 /* Free list to use to allocate new buffers. */
491 /* CLIB fifo of buffer indices waiting to be unserialized. */
494 /* Event type used to signal that RX buffers have been added to fifo. */
495 uword ready_one_time_event;
498 } vlib_serialize_buffer_main_t;
500 void serialize_open_vlib_buffer (serialize_main_t * m, struct vlib_main_t *vm,
501 vlib_serialize_buffer_main_t * sm);
502 void unserialize_open_vlib_buffer (serialize_main_t * m,
503 struct vlib_main_t *vm,
504 vlib_serialize_buffer_main_t * sm);
506 u32 serialize_close_vlib_buffer (serialize_main_t * m);
507 void unserialize_close_vlib_buffer (serialize_main_t * m);
508 void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
511 serialize_vlib_buffer_n_bytes (serialize_main_t * m)
513 serialize_stream_t *s = &m->stream;
514 vlib_serialize_buffer_main_t *sm
515 = uword_to_pointer (m->stream.data_function_opaque,
516 vlib_serialize_buffer_main_t *);
517 return sm->tx.n_total_data_bytes + s->current_buffer_index +
518 vec_len (s->overflow_buffer);
524 /** \brief Compile time buffer trajectory tracing option
525 Turn this on if you run into "bad monkey" contexts,
526 and you want to know exactly which nodes they've visited...
529 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
531 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
532 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
533 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
534 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
535 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
536 vlib_buffer_trace_trajectory_init (b);
538 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
539 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
541 #endif /* included_vlib_buffer_h */
543 #define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \
544 __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \
545 static void __vlib_add_buffer_callbacks_t_##x (void) \
546 __attribute__((__constructor__)) ; \
547 static void __vlib_add_buffer_callbacks_t_##x (void) \
549 if (vlib_buffer_callbacks) \
550 clib_panic ("vlib buffer callbacks already registered"); \
551 vlib_buffer_callbacks = &__##x##_buffer_callbacks; \
553 __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
556 * fd.io coding-style-patch-verification: ON
559 * eval: (c-set-style "gnu")