#include <vlib/error.h> /* for vlib_error_t */
#include <vlib/config.h> /* for __PRE_DATA_SIZE */
-#define VLIB_BUFFER_DATA_SIZE (2048)
#define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
+#define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
+
/* Minimum buffer chain segment size. Does not apply to last buffer in chain.
Dataplane code can safely asume that specified amount of data is not split
into 2 chained buffers */
/* Amount of head buffer data copied to each replica head buffer */
#define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
-typedef u8 vlib_buffer_free_list_index_t;
-
/** \file
vlib buffer structure definition and a few select
access methods. This structure and the buffer allocation
*/
#define foreach_vlib_buffer_flag \
_( 0, IS_TRACED, 0) \
- _( 1, NEXT_PRESENT, 0) \
+ _( 1, NEXT_PRESENT, "next-present") \
_( 2, TOTAL_LENGTH_VALID, 0) \
_( 3, EXT_HDR_VALID, "ext-hdr-valid")
/** Generic flow identifier */
u32 flow_id;
- /** Number of additional references to this buffer. */
- u8 n_add_refs;
+ /** Reference count for this buffer. */
+ volatile u8 ref_count;
/** index of buffer pool this buffer belongs. */
u8 buffer_pool_index;
* VLIB_BUFFER_NEXT_PRESENT flag is set. */
u32 next_buffer;
- /** Used by feature subgraph arcs to visit enabled feature nodes */
- u32 current_config_index;
+ /** The following fields can be in a union because once a packet enters
+ * the punt path, it is no longer on a feature arc */
+ union
+ {
+ /** Used by feature subgraph arcs to visit enabled feature nodes */
+ u32 current_config_index;
+ /* the reason the packet once punted */
+ u32 punt_reason;
+ };
/** Opaque data used by sub-graphs for their own purposes. */
u32 opaque[10];
/** start of 2nd cache line */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
- /** Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is
+ /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
* set. */
- u32 trace_index;
+ u32 trace_handle;
/** Only valid for first buffer in chain. Current length plus total length
* given here give total number of bytes in buffer chain. */
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE];
/** Packet data */
- u8 data[0];
+ u8 data[];
};
#ifdef CLIB_HAVE_VEC128
u8x16 as_u8x16[4];
always_inline void *
vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
{
- ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
b->current_data += size;
return vlib_buffer_get_current (b);
}
+/** \brief Construct a trace handle from thread and pool index
+ * @param thread Thread id
+ * @param pool_index Pool index
+ * @return trace handle
+ */
+always_inline u32
+vlib_buffer_make_trace_handle (u32 thread, u32 pool_index)
+{
+ u32 rv;
+ ASSERT (thread < 0xff);
+ ASSERT (pool_index < 0x00FFFFFF);
+ rv = (thread << 24) | (pool_index & 0x00FFFFFF);
+ return rv;
+}
+
+/** \brief Extract the thread id from a trace handle
+ * @param trace_handle the trace handle
+ * @return the thread id
+ */
+always_inline u32
+vlib_buffer_get_trace_thread (vlib_buffer_t * b)
+{
+ u32 trace_handle = b->trace_handle;
+
+ return trace_handle >> 24;
+}
+
+/** \brief Extract the trace (pool) index from a trace handle
+ * @param trace_handle the trace handle
+ * @return the trace index
+ */
+always_inline u32
+vlib_buffer_get_trace_index (vlib_buffer_t * b)
+{
+ u32 trace_handle = b->trace_handle;
+ return trace_handle & 0x00FFFFFF;
+}
+
/** \brief Retrieve bytes from buffer head
* @param b pointer to the buffer
* @param size number of bytes to pull
/* Forward declaration. */
struct vlib_main_t;
-typedef struct vlib_buffer_free_list_t
-{
- /* Template buffer used to initialize first 16 bytes of buffers
- allocated on this free list. */
- vlib_buffer_t buffer_init_template;
-
- /* Our index into vlib_main_t's buffer_free_list_pool. */
- vlib_buffer_free_list_index_t index;
-
- /* Number of buffers to allocate when we need to allocate new buffers */
- u32 min_n_buffers_each_alloc;
-
- /* Total number of buffers allocated from this free list. */
- u32 n_alloc;
-
- /* Vector of free buffers. Each element is a byte offset into I/O heap. */
- u32 *buffers;
-
- /* index of buffer pool used to get / put buffers */
- u8 buffer_pool_index;
-
- /* Free list name. */
- u8 *name;
-
- /* Callback functions to initialize newly allocated buffers.
- If null buffers are zeroed. */
- void (*buffer_init_function) (struct vlib_main_t * vm,
- struct vlib_buffer_free_list_t * fl,
- u32 * buffers, u32 n_buffers);
-
- uword buffer_init_function_opaque;
-} __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
-
-typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm,
- vlib_buffer_free_list_t * fl,
- uword min_free_buffers);
-typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers,
- u32 n_buffers);
-typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
- u32 * buffers, u32 n_buffers);
+#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
typedef struct
{
- vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb;
- vlib_buffer_free_cb_t *vlib_buffer_free_cb;
- vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb;
-} vlib_buffer_callbacks_t;
-
-extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ];
+ u32 n_cached;
+} vlib_buffer_pool_thread_t;
typedef struct
{
uword start;
uword size;
uword log2_page_size;
+ u8 index;
+ u32 numa_node;
u32 physmem_map_index;
- u32 buffer_size;
+ u32 data_size;
+ u32 n_buffers;
+ u32 n_avail;
u32 *buffers;
+ u8 *name;
clib_spinlock_t lock;
+
+ /* per-thread data */
+ vlib_buffer_pool_thread_t *threads;
+
+ /* buffer metadata template */
+ vlib_buffer_t buffer_template;
} vlib_buffer_pool_t;
+#define VLIB_BUFFER_MAX_NUMA_NODES 32
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
uword buffer_mem_size;
vlib_buffer_pool_t *buffer_pools;
- /* Buffer free callback, for subversive activities */
- u32 (*buffer_free_callback) (struct vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers, u32 follow_buffer_next);
-#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
-
- /* Hash table mapping buffer size (rounded to next unit of
- sizeof (vlib_buffer_t)) to free list index. */
- uword *free_list_by_size;
-
/* Hash table mapping buffer index into number
0 => allocated but free, 1 => allocated and not-free.
If buffer index is not in hash table then this buffer
has never been allocated. */
uword *buffer_known_hash;
clib_spinlock_t buffer_known_hash_lockp;
+ u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
- /* Callbacks */
- vlib_buffer_callbacks_t cb;
- int callbacks_registered;
-} vlib_buffer_main_t;
+ /* config */
+ u32 buffers_per_numa;
+ u16 ext_hdr_size;
+ u32 default_data_size;
-u8 vlib_buffer_register_physmem_map (struct vlib_main_t *vm,
- u32 physmem_map_index);
+ /* logging */
+ vlib_log_class_t log_default;
+} vlib_buffer_main_t;
clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
-
-void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
-
/*
*/
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
#endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
-#endif /* included_vlib_buffer_h */
+extern u16 __vlib_buffer_external_hdr_size;
+#define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
+static void __clib_constructor \
+vnet_buffer_set_ext_hdr_size() \
+{ \
+ if (__vlib_buffer_external_hdr_size) \
+ clib_error ("buffer external header space already set"); \
+ __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
+}
-#define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \
- __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \
-static void __vlib_add_buffer_callbacks_t_##x (void) \
- __attribute__((__constructor__)) ; \
-static void __vlib_add_buffer_callbacks_t_##x (void) \
-{ \
- if (vlib_buffer_callbacks) \
- clib_panic ("vlib buffer callbacks already registered"); \
- vlib_buffer_callbacks = &__##x##_buffer_callbacks; \
-} \
-static void __vlib_rm_buffer_callbacks_t_##x (void) \
- __attribute__((__destructor__)) ; \
-static void __vlib_rm_buffer_callbacks_t_##x (void) \
-{ vlib_buffer_callbacks = 0; } \
-__VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
+#endif /* included_vlib_buffer_h */
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff