struct vlib_node_runtime_t * node,
struct vlib_frame_t * frame);
+typedef enum
+{
+ VLIB_NODE_PROTO_HINT_NONE = 0,
+ VLIB_NODE_PROTO_HINT_ETHERNET,
+ VLIB_NODE_PROTO_HINT_IP4,
+ VLIB_NODE_PROTO_HINT_IP6,
+ VLIB_NODE_PROTO_HINT_TCP,
+ VLIB_NODE_PROTO_HINT_UDP,
+ VLIB_NODE_N_PROTO_HINTS,
+} vlib_node_proto_hint_t;
+
typedef enum
{
/* An internal node on the call graph (could be output). */
vlib_node_function_t *function;
int priority;
struct _vlib_node_fn_registration *next_registration;
+ char *name;
} vlib_node_fn_registration_t;
typedef struct _vlib_node_registration
/* Error strings indexed by error code for this node. */
char **error_strings;
+ vl_counter_t *error_counters;
/* Buffer format/unformat for this node. */
format_function_t *format_buffer;
/* Node flags. */
u16 flags;
+ /* protocol at b->data[b->current_data] upon entry to the dispatch fn */
+ u8 protocol_hint;
+
/* Size of scalar and vector arguments in bytes. */
u16 scalar_size, vector_size;
} vlib_node_registration_t;
+#ifndef CLIB_MARCH_VARIANT
#define VLIB_REGISTER_NODE(x,...) \
__VA_ARGS__ vlib_node_registration_t x; \
static void __vlib_add_node_registration_##x (void) \
&x, next_registration); \
} \
__VA_ARGS__ vlib_node_registration_t x
+#else
+#define VLIB_REGISTER_NODE(x,...) \
+STATIC_ASSERT (sizeof(# __VA_ARGS__) != 7,"node " #x " must not be declared as static"); \
+static __clib_unused vlib_node_registration_t __clib_unused_##x
+#endif
+
+#ifndef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_VARIANT_STR "default"
+#else
+#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s)
+#define __CLIB_MARCH_VARIANT_STR(s) #s
+#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT)
+#endif
#define VLIB_NODE_FN(node) \
uword CLIB_MARCH_SFX (node##_fn)(); \
vlib_node_fn_registration_t *r; \
r = & CLIB_MARCH_SFX (node##_fn_registration); \
r->priority = CLIB_MARCH_FN_PRIORITY(); \
+ r->name = CLIB_MARCH_VARIANT_STR; \
r->next_registration = node.node_fn_registrations; \
node.node_fn_registrations = r; \
} \
uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn)
-#if CLIB_DEBUG > 0
-#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn)
-#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn)
-#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn)
-#else
-#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \
- uword \
- __attribute__ ((flatten)) \
- __attribute__ ((target (tgt))) \
- CLIB_CPU_OPTIMIZED \
- fn ## _ ## arch ( struct vlib_main_t * vm, \
- struct vlib_node_runtime_t * node, \
- struct vlib_frame_t * frame) \
- { return fn (vm, node, frame); }
-
-#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \
- foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn)
-
-#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \
- VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \
- CLIB_MULTIARCH_SELECT_FN(fn, static inline) \
- static void __attribute__((__constructor__)) \
- __vlib_node_function_multiarch_select_##node (void) \
- { node.function = fn ## _multiarch_select(); }
-#endif
+unformat_function_t unformat_vlib_node_variant;
always_inline vlib_node_registration_t *
vlib_node_next_registered (vlib_node_registration_t * c)
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE (1 << 6)
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE (1 << 7)
+#define VLIB_NODE_FLAG_TRACE_SUPPORTED (1 << 8)
/* State for input nodes. */
u8 state;
/* Number of bytes of run time data. */
u8 runtime_data_bytes;
+ /* protocol at b->data[b->current_data] upon entry to the dispatch fn */
+ u8 protocol_hint;
+
/* Number of error codes used by this node. */
u16 n_errors;
u32 error_heap_handle;
u32 error_heap_index;
- /* Error strings indexed by error code for this node. */
- char **error_strings;
+ /* Counter structures indexed by counter code for this node. */
+ vl_counter_t *error_counters;
/* Vector of next node names.
Only used before next_nodes array is initialized. */
struct vlib_frame_t * f);
/* for pretty-printing, not typically valid */
u8 *state_string;
+
+ /* Node function candidate registration with priority */
+ vlib_node_fn_registration_t *node_fn_registrations;
} vlib_node_t;
#define VLIB_INVALID_NODE_INDEX ((u32) ~0)
typedef struct vlib_frame_t
{
/* Frame flags. */
+ u16 frame_flags;
+
+ /* User flags. Used for sending hints to the next node. */
u16 flags;
/* Number of scalar bytes in arguments. */
typedef struct
{
- /* Frame index. */
- u32 frame_index;
+ /* Frame pointer. */
+ vlib_frame_t *frame;
/* Node runtime for this next. */
u32 node_runtime_index;
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \
VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
+ /* Don't append this frame */
+#define VLIB_FRAME_NO_APPEND (1 << 14)
+
/* This next frame owns enqueue to node
corresponding to node_runtime_index. */
#define VLIB_FRAME_OWNER (1 << 15)
always_inline void
vlib_next_frame_init (vlib_next_frame_t * nf)
{
- memset (nf, 0, sizeof (nf[0]));
- nf->frame_index = ~0;
+ clib_memset (nf, 0, sizeof (nf[0]));
nf->node_runtime_index = ~0;
}
u32 node_runtime_index;
/* Frame index (in the heap). */
- u32 frame_index;
+ vlib_frame_t *frame;
/* Start of next frames for this node. */
u32 next_frame_index;
vlib_error_t *errors; /**< Vector of errors for this node. */
-#if __SIZEOF_POINTER__ == 4
- u8 pad[8];
-#endif
-
u32 clocks_since_last_overflow; /**< Number of clock cycles. */
u32 max_clock; /**< Maximum clock cycle for an
u16 state; /**< Input node state. */
+ u32 interrupt_data; /**< Data passed together with interrupt.
+ Valid only when state is
+ VLIB_NODE_STATE_INTERRUPT */
+
u16 n_next_nodes;
u16 cached_next_index; /**< Next frame index that vector
/* Number of allocated frames for this scalar/vector size. */
u32 n_alloc_frames;
- /* Vector of free frame indices for this scalar/vector size. */
- u32 *free_frame_indices;
+ /* Vector of free frames for this scalar/vector size. */
+ vlib_frame_t **free_frames;
} vlib_frame_size_t;
typedef struct
typedef struct
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
/* Node runtime for this process. */
vlib_node_runtime_t node_runtime;
vlib_cli_output_function_t *output_function;
uword output_function_arg;
-#ifdef CLIB_UNIX
- /* Pad to a multiple of the page size so we can mprotect process stacks */
-#define PAGE_SIZE_MULTIPLE 0x1000
-#define ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT __attribute__ ((aligned (PAGE_SIZE_MULTIPLE)))
-#else
-#define ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT
-#endif
-
- /* Process stack. Starts here and extends 2^log2_n_stack_bytes
- bytes. */
-
+ /* Process stack */
#define VLIB_PROCESS_STACK_MAGIC (0xdead7ead)
- u32 stack[0] ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT;
-} vlib_process_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
-
-#ifdef CLIB_UNIX
- /* Ensure that the stack is aligned on the multiple of the page size */
-typedef char
- assert_process_stack_must_be_aligned_exactly_to_page_size_multiple[(sizeof
- (vlib_process_t)
- -
- PAGE_SIZE_MULTIPLE)
- ==
- 0 ? 0 :
- -1];
-#endif
+ u32 *stack;
+} vlib_process_t;
typedef struct
{
return d / 2;
}
+typedef struct
+{
+ u32 node_runtime_index;
+ u32 data;
+} vlib_node_interrupt_t;
+
typedef struct
{
/* Public nodes. */
vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE];
/* Node runtime indices for input nodes with pending interrupts. */
- u32 *pending_interrupt_node_runtime_indices;
+ vlib_node_interrupt_t *pending_local_interrupts;
+ vlib_node_interrupt_t *pending_remote_interrupts;
+ volatile u32 *pending_remote_interrupts_notify;
clib_spinlock_t pending_interrupt_lock;
/* Input nodes are switched from/to interrupt to/from polling mode
/* Node registrations added by constructors */
vlib_node_registration_t *node_registrations;
+
+ /* Node index from error code */
+ u32 *node_by_error;
} vlib_node_main_t;
+typedef u16 vlib_error_t;
+
+always_inline u32
+vlib_error_get_node (vlib_node_main_t * nm, vlib_error_t e)
+{
+ return nm->node_by_error[e];
+}
+
+always_inline u32
+vlib_error_get_code (vlib_node_main_t * nm, vlib_error_t e)
+{
+ u32 node_index = nm->node_by_error[e];
+ vlib_node_t *n = nm->nodes[node_index];
+ u32 error_code = e - n->error_heap_index;
+ return error_code;
+}
-#define FRAME_QUEUE_MAX_NELTS 32
+#define FRAME_QUEUE_MAX_NELTS 64
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Code Review / vpp.git / blobdiff