*/
#define foreach_vlib_buffer_flag \
_( 0, IS_TRACED, 0) \
- _( 1, NEXT_PRESENT, 0) \
+ _( 1, NEXT_PRESENT, "next-present") \
_( 2, TOTAL_LENGTH_VALID, 0) \
_( 3, EXT_HDR_VALID, "ext-hdr-valid")
#define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
#define VLIB_BUFFER_FLAGS_ALL (0x0f)
+/** \brief Compile time buffer trajectory tracing option
+ Turn this on if you run into "bad monkey" contexts,
+ and you want to know exactly which nodes they've visited...
+ See vlib/main.c...
+*/
+#ifndef VLIB_BUFFER_TRACE_TRAJECTORY
+#define VLIB_BUFFER_TRACE_TRAJECTORY 0
+#endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
+
+#define vlib_buffer_template_fields \
+ /** signed offset in data[], pre_data[] that we are currently \
+ * processing. If negative current header points into predata area. */ \
+ i16 current_data; \
+ \
+ /** Nbytes between current data and the end of this buffer. */ \
+ u16 current_length; \
+ /** buffer flags: \
+ <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list \
+ index, <br> VLIB_BUFFER_IS_TRACED: trace this buffer. <br> \
+ VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer. <br> \
+ VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says <br> \
+ VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager \
+ header, set to avoid adding it to a flow report <br> \
+ VLIB_BUFFER_FLAG_USER(n): user-defined bit N \
+ */ \
+ u32 flags; \
+ \
+ /** Generic flow identifier */ \
+ u32 flow_id; \
+ \
+ /** Reference count for this buffer. */ \
+ volatile u8 ref_count; \
+ \
+ /** index of buffer pool this buffer belongs. */ \
+ u8 buffer_pool_index; \
+ \
+ /** Error code for buffers to be enqueued to error handler. */ \
+ vlib_error_t error; \
+ \
+ /** Next buffer for this linked-list of buffers. Only valid if \
+ * VLIB_BUFFER_NEXT_PRESENT flag is set. */ \
+ u32 next_buffer; \
+ \
+ /** The following fields can be in a union because once a packet enters \
+ * the punt path, it is no longer on a feature arc */ \
+ union \
+ { \
+ /** Used by feature subgraph arcs to visit enabled feature nodes */ \
+ u32 current_config_index; \
+ /* the reason the packet once punted */ \
+ u32 punt_reason; \
+ }; \
+ \
+ /** Opaque data used by sub-graphs for their own purposes. */ \
+ u32 opaque[10];
+
+typedef struct
+{
+ CLIB_ALIGN_MARK (align_mark, 64);
+ vlib_buffer_template_fields
+} vlib_buffer_template_t;
+
+STATIC_ASSERT_SIZEOF (vlib_buffer_template_t, 64);
+
/** VLIB buffer representation. */
typedef union
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
struct
{
- CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-
- /** signed offset in data[], pre_data[] that we are currently
- * processing. If negative current header points into predata area. */
- i16 current_data;
-
- /** Nbytes between current data and the end of this buffer. */
- u16 current_length;
-
- /** buffer flags:
- <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
- <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
- <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
- <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
- <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
- set to avoid adding it to a flow report
- <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
- */
- u32 flags;
-
- /** Generic flow identifier */
- u32 flow_id;
-
- /** Reference count for this buffer. */
- volatile u8 ref_count;
-
- /** index of buffer pool this buffer belongs. */
- u8 buffer_pool_index;
-
- /** Error code for buffers to be enqueued to error handler. */
- vlib_error_t error;
-
- /** Next buffer for this linked-list of buffers. Only valid if
- * VLIB_BUFFER_NEXT_PRESENT flag is set. */
- u32 next_buffer;
-
- /** The following fields can be in a union because once a packet enters
- * the punt path, it is no longer on a feature arc */
union
{
- /** Used by feature subgraph arcs to visit enabled feature nodes */
- u32 current_config_index;
- /* the reason the packet once punted */
- u32 punt_reason;
+ struct
+ {
+ vlib_buffer_template_fields
+ };
+ vlib_buffer_template_t template;
};
- /** Opaque data used by sub-graphs for their own purposes. */
- u32 opaque[10];
+ /* Data above is initialized or zeroed on alloc, data bellow is not
+ * and it is app responsibility to ensure data is valid */
- /** part of buffer metadata which is initialized on alloc ends here. */
- STRUCT_MARK (template_end);
-
- /** start of 2nd cache line */
- CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+ /** start of 2nd half (2nd cacheline on systems where cacheline size is 64) */
+ CLIB_ALIGN_MARK (second_half, 64);
/** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
* set. */
/**< More opaque data, see ../vnet/vnet/buffer.h */
u32 opaque2[14];
- /** start of third cache line */
- CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
+#if VLIB_BUFFER_TRACE_TRAJECTORY > 0
+ /** trace trajectory data - we use a specific cacheline for that in the
+ * buffer when it is compiled-in */
+#define VLIB_BUFFER_TRACE_TRAJECTORY_MAX 31
+#define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 64
+#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->trajectory_nb = 0
+ CLIB_ALIGN_MARK (trajectory, 64);
+ u16 trajectory_nb;
+ u16 trajectory_trace[VLIB_BUFFER_TRACE_TRAJECTORY_MAX];
+#else /* VLIB_BUFFER_TRACE_TRAJECTORY */
+#define VLIB_BUFFER_TRACE_TRAJECTORY_SZ 0
+#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
+#endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
+
+ /** start of buffer headroom */
+ CLIB_ALIGN_MARK (headroom, 64);
/** Space for inserting data before buffer start. Packet rewrite string
* will be rewritten backwards and may extend back before
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE];
/** Packet data */
- u8 data[0];
+ u8 data[];
};
#ifdef CLIB_HAVE_VEC128
u8x16 as_u8x16[4];
#endif
} vlib_buffer_t;
+STATIC_ASSERT_SIZEOF (vlib_buffer_t, 128 + VLIB_BUFFER_TRACE_TRAJECTORY_SZ +
+ VLIB_BUFFER_PRE_DATA_SIZE);
+STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE % CLIB_CACHE_LINE_BYTES == 0,
+ "VLIB_BUFFER_PRE_DATA_SIZE must be divisible by cache line size");
+
#define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
+#define VLIB_BUFFER_INVALID_INDEX 0xffffffff
/** \brief Prefetch buffer metadata.
The first 64 bytes of buffer contains most header information
always_inline void *
vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size)
{
- ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size);
b->current_data += size;
return vlib_buffer_get_current (b);
}
/* Forward declaration. */
struct vlib_main_t;
+#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- u32 *cached_buffers;
- u32 n_alloc;
+ u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ];
+ u32 n_cached;
} vlib_buffer_pool_thread_t;
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
uword start;
uword size;
- uword log2_page_size;
+ u8 log2_page_size;
u8 index;
- u32 numa_node;
+ u8 numa_node;
u32 physmem_map_index;
u32 data_size;
+ u32 alloc_size;
u32 n_buffers;
+ u32 n_avail;
u32 *buffers;
u8 *name;
clib_spinlock_t lock;
vlib_buffer_pool_thread_t *threads;
/* buffer metadata template */
- vlib_buffer_t buffer_template;
+ vlib_buffer_template_t buffer_template;
} vlib_buffer_pool_t;
#define VLIB_BUFFER_MAX_NUMA_NODES 32
+typedef u32 (vlib_buffer_alloc_free_callback_t) (struct vlib_main_t *vm,
+ u8 buffer_pool_index,
+ u32 *buffers, u32 n_buffers);
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
uword buffer_mem_size;
vlib_buffer_pool_t *buffer_pools;
- /* Hash table mapping buffer index into number
- 0 => allocated but free, 1 => allocated and not-free.
- If buffer index is not in hash table then this buffer
- has never been allocated. */
- uword *buffer_known_hash;
- clib_spinlock_t buffer_known_hash_lockp;
+ vlib_buffer_alloc_free_callback_t *alloc_callback_fn;
+ vlib_buffer_alloc_free_callback_t *free_callback_fn;
+
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
/* config */
u32 buffers_per_numa;
u16 ext_hdr_size;
u32 default_data_size;
+ clib_mem_page_sz_t log2_page_size;
+
+ /* Hash table mapping buffer index into number
+ 0 => allocated but free, 1 => allocated and not-free.
+ If buffer index is not in hash table then this buffer
+ has never been allocated. */
+ uword *buffer_known_hash;
+ clib_spinlock_t buffer_known_hash_lockp;
/* logging */
vlib_log_class_t log_default;
clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
-/*
- */
-
-/** \brief Compile time buffer trajectory tracing option
- Turn this on if you run into "bad monkey" contexts,
- and you want to know exactly which nodes they've visited...
- See vlib/main.c...
-*/
-#define VLIB_BUFFER_TRACE_TRAJECTORY 0
+format_function_t format_vlib_buffer_pool_all;
-#if VLIB_BUFFER_TRACE_TRAJECTORY > 0
-extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
-extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
-extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
-#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
- vlib_buffer_trace_trajectory_init (b);
-#else
-#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
-#endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
+int vlib_buffer_set_alloc_free_callback (
+ struct vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
+ vlib_buffer_alloc_free_callback_t *free_callback_fn);
extern u16 __vlib_buffer_external_hdr_size;
#define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
Code Review / vpp.git / blobdiff