#include <vppinfra/cpu.h>
#include <vppinfra/longjmp.h>
-#include <vppinfra/timing_wheel.h>
+#include <vppinfra/lock.h>
#include <vlib/trace.h> /* for vlib_trace_filter_t */
/* Forward declaration. */
struct vlib_node_runtime_t * node,
struct vlib_frame_t * frame);
+typedef enum
+{
+ VLIB_NODE_PROTO_HINT_NONE = 0,
+ VLIB_NODE_PROTO_HINT_ETHERNET,
+ VLIB_NODE_PROTO_HINT_IP4,
+ VLIB_NODE_PROTO_HINT_IP6,
+ VLIB_NODE_PROTO_HINT_TCP,
+ VLIB_NODE_PROTO_HINT_UDP,
+ VLIB_NODE_N_PROTO_HINTS,
+} vlib_node_proto_hint_t;
+
typedef enum
{
/* An internal node on the call graph (could be output). */
VLIB_N_NODE_TYPE,
} vlib_node_type_t;
+typedef struct _vlib_node_fn_registration
+{
+ vlib_node_function_t *function;
+ int priority;
+ struct _vlib_node_fn_registration *next_registration;
+ char *name;
+} vlib_node_fn_registration_t;
+
typedef struct _vlib_node_registration
{
/* Vector processing function for this node. */
vlib_node_function_t *function;
+ /* Node function candidate registration with priority */
+ vlib_node_fn_registration_t *node_fn_registrations;
+
/* Node name. */
char *name;
/* Node flags. */
u16 flags;
+ /* protocol at b->data[b->current_data] upon entry to the dispatch fn */
+ u8 protocol_hint;
+
/* Size of scalar and vector arguments in bytes. */
u16 scalar_size, vector_size;
} vlib_node_registration_t;
+#ifndef CLIB_MARCH_VARIANT
#define VLIB_REGISTER_NODE(x,...) \
__VA_ARGS__ vlib_node_registration_t x; \
static void __vlib_add_node_registration_##x (void) \
x.next_registration = vm->node_main.node_registrations; \
vm->node_main.node_registrations = &x; \
} \
+static void __vlib_rm_node_registration_##x (void) \
+ __attribute__((__destructor__)) ; \
+static void __vlib_rm_node_registration_##x (void) \
+{ \
+ vlib_main_t * vm = vlib_get_main(); \
+ VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations, \
+ &x, next_registration); \
+} \
__VA_ARGS__ vlib_node_registration_t x
+#else
+#define VLIB_REGISTER_NODE(x,...) \
+static __clib_unused vlib_node_registration_t __clib_unused_##x
+#endif
+
+#ifndef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_VARIANT_STR "default"
+#else
+#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s)
+#define __CLIB_MARCH_VARIANT_STR(s) #s
+#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT)
+#endif
+
+#define VLIB_NODE_FN(node) \
+uword CLIB_MARCH_SFX (node##_fn)(); \
+static vlib_node_fn_registration_t \
+ CLIB_MARCH_SFX(node##_fn_registration) = \
+ { .function = &CLIB_MARCH_SFX (node##_fn), }; \
+ \
+static void __clib_constructor \
+CLIB_MARCH_SFX (node##_multiarch_register) (void) \
+{ \
+ extern vlib_node_registration_t node; \
+ vlib_node_fn_registration_t *r; \
+ r = & CLIB_MARCH_SFX (node##_fn_registration); \
+ r->priority = CLIB_MARCH_FN_PRIORITY(); \
+ r->name = CLIB_MARCH_VARIANT_STR; \
+ r->next_registration = node.node_fn_registrations; \
+ node.node_fn_registrations = r; \
+} \
+uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn)
#if CLIB_DEBUG > 0
#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn)
u64 calls, vectors, clocks, suspends;
u64 max_clock;
u64 max_clock_n;
+ u64 perf_counter_ticks;
+ u64 perf_counter_vectors;
} vlib_node_stats_t;
#define foreach_vlib_node_state \
/* Number of bytes of run time data. */
u8 runtime_data_bytes;
+ /* protocol at b->data[b->current_data] upon entry to the dispatch fn */
+ u8 protocol_hint;
+
/* Number of error codes used by this node. */
u16 n_errors;
struct vlib_frame_t * f);
/* for pretty-printing, not typically valid */
u8 *state_string;
+
+ /* Node function candidate registration with priority */
+ vlib_node_fn_registration_t *node_fn_registrations;
} vlib_node_t;
#define VLIB_INVALID_NODE_INDEX ((u32) ~0)
/* Max number of vector elements to process at once per node. */
#define VLIB_FRAME_SIZE 256
-#define VLIB_FRAME_ALIGN VLIB_MAX_CPUS
+#define VLIB_FRAME_ALIGN CLIB_CACHE_LINE_BYTES
/* Calling frame (think stack frame) for a node. */
typedef struct vlib_frame_t
{
/* Frame flags. */
+ u16 frame_flags;
+
+ /* User flags. Used for sending hints to the next node. */
u16 flags;
/* Number of scalar bytes in arguments. */
/* Number of vector elements currently in frame. */
u16 n_vectors;
- /* Owner cpuid / heap id */
- u16 cpu_index;
-
/* Scalar and vector arguments to next node. */
u8 arguments[0];
} vlib_frame_t;
always_inline void
vlib_next_frame_init (vlib_next_frame_t * nf)
{
- memset (nf, 0, sizeof (nf[0]));
+ clib_memset (nf, 0, sizeof (nf[0]));
nf->frame_index = ~0;
nf->node_runtime_index = ~0;
}
typedef struct vlib_node_runtime_t
{
- CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- /* Node function to call. */
- vlib_node_function_t *function;
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /**< cacheline mark */
- /* Vector of errors for this node. */
- vlib_error_t *errors;
+ vlib_node_function_t *function; /**< Node function to call. */
- /* Number of clock cycles. */
- u32 clocks_since_last_overflow;
+ vlib_error_t *errors; /**< Vector of errors for this node. */
- /* Maximum clock cycle for an invocation. */
- u32 max_clock;
+#if __SIZEOF_POINTER__ == 4
+ u8 pad[8];
+#endif
- /* Number of vectors in the recorded max_clock. */
- u32 max_clock_n;
+ u32 clocks_since_last_overflow; /**< Number of clock cycles. */
- /* Number of calls. */
- u32 calls_since_last_overflow;
+ u32 max_clock; /**< Maximum clock cycle for an
+ invocation. */
- /* Number of vector elements processed by this node. */
- u32 vectors_since_last_overflow;
+ u32 max_clock_n; /**< Number of vectors in the recorded
+ max_clock. */
- /* Start of next frames for this node. */
- u32 next_frame_index;
+ u32 calls_since_last_overflow; /**< Number of calls. */
- /* Node index. */
- u32 node_index;
+ u32 vectors_since_last_overflow; /**< Number of vector elements
+ processed by this node. */
- /* For input nodes: decremented on each main loop interation until it reaches zero
- and function is called. Allows some input nodes to be called
- more than others. */
- u32 input_main_loops_per_call;
+ u32 perf_counter_ticks_since_last_overflow; /**< Perf counter ticks */
+ u32 perf_counter_vectors_since_last_overflow; /**< Perf counter vectors */
- /* Saved main loop counter of last dispatch of this node. */
- u32 main_loop_count_last_dispatch;
+ u32 next_frame_index; /**< Start of next frames for this
+ node. */
- u32 main_loop_vector_stats[2];
+ u32 node_index; /**< Node index. */
- /* Copy of main node flags. */
- u16 flags;
+ u32 input_main_loops_per_call; /**< For input nodes: decremented
+ on each main loop interation until
+ it reaches zero and function is
+ called. Allows some input nodes to
+ be called more than others. */
- /* Input node state. */
- u16 state;
+ u32 main_loop_count_last_dispatch; /**< Saved main loop counter of last
+ dispatch of this node. */
- u16 n_next_nodes;
+ u32 main_loop_vector_stats[2];
- /* Next frame index that vector arguments were last enqueued to
- last time this node ran. Set to zero before first run
- of this node. */
- u16 cached_next_index;
+ u16 flags; /**< Copy of main node flags. */
- /* CPU this node runs on */
- u16 cpu_index;
+ u16 state; /**< Input node state. */
- /* Function dependent node-runtime. */
- u8 runtime_data[0];
+ u16 n_next_nodes;
+
+ u16 cached_next_index; /**< Next frame index that vector
+ arguments were last enqueued to
+ last time this node ran. Set to
+ zero before first run of this
+ node. */
+
+ u16 thread_index; /**< thread this node runs on */
+
+ u8 runtime_data[0]; /**< Function dependent
+ node-runtime data. This data is
+ thread local, and it is not
+ cloned from main thread. It needs
+ to be initialized for each thread
+ before it is used unless
+ runtime_data template exists in
+ vlib_node_t. */
}
vlib_node_runtime_t;
+#define VLIB_NODE_RUNTIME_DATA_SIZE (sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data))
+
typedef struct
{
/* Number of allocated frames for this scalar/vector size. */
/* Pool of currently valid event types. */
vlib_process_event_type_t *event_type_pool;
- /* When suspending saves cpu cycle counter when process is to be resumed. */
- u64 resume_cpu_time;
+ /*
+ * When suspending saves clock time (10us ticks) when process
+ * is to be resumed.
+ */
+ u64 resume_clock_interval;
+
+ /* Handle from timer code, to cancel an unexpired timer */
+ u32 stop_timer_handle;
/* Default output function and its argument for any CLI outputs
within the process. */
/* Node runtime indices for input nodes with pending interrupts. */
u32 *pending_interrupt_node_runtime_indices;
+ clib_spinlock_t pending_interrupt_lock;
/* Input nodes are switched from/to interrupt to/from polling mode
when average vector length goes above/below polling/interrupt
vlib_pending_frame_t *pending_frames;
/* Timing wheel for scheduling time-based node dispatch. */
- timing_wheel_t timing_wheel;
+ void *timing_wheel;
vlib_signal_timed_event_data_t *signal_timed_event_data_pool;
u32 *data_from_advancing_timing_wheel;
/* CPU time of next process to be ready on timing wheel. */
- u64 cpu_time_next_process_ready;
+ f64 time_next_process_ready;
/* Vector of process nodes.
One for each node of type VLIB_NODE_TYPE_PROCESS. */
Code Review / vpp.git / blobdiff