X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode.h;h=f41eb60aa2c4284d5063f638570ed0a15e5211b6;hb=7fff3d205463d5e0a95d6bdd337100988ef323a3;hp=b624e9d636d2d55e00f0ad6e58c7cde8e0b96484;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git diff --git a/src/vlib/node.h b/src/vlib/node.h index b624e9d636d..f41eb60aa2c 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -42,7 +42,7 @@ #include #include -#include +#include #include /* for vlib_trace_filter_t */ /* Forward declaration. */ @@ -55,6 +55,17 @@ typedef uword (vlib_node_function_t) (struct vlib_main_t * vm, struct vlib_node_runtime_t * node, struct vlib_frame_t * frame); +typedef enum +{ + VLIB_NODE_PROTO_HINT_NONE = 0, + VLIB_NODE_PROTO_HINT_ETHERNET, + VLIB_NODE_PROTO_HINT_IP4, + VLIB_NODE_PROTO_HINT_IP6, + VLIB_NODE_PROTO_HINT_TCP, + VLIB_NODE_PROTO_HINT_UDP, + VLIB_NODE_N_PROTO_HINTS, +} vlib_node_proto_hint_t; + typedef enum { /* An internal node on the call graph (could be output). */ @@ -75,11 +86,22 @@ typedef enum VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct _vlib_node_fn_registration +{ + vlib_node_function_t *function; + int priority; + struct _vlib_node_fn_registration *next_registration; + char *name; +} vlib_node_fn_registration_t; + typedef struct _vlib_node_registration { /* Vector processing function for this node. */ vlib_node_function_t *function; + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; + /* Node name. */ char *name; @@ -123,6 +145,9 @@ typedef struct _vlib_node_registration /* Node flags. */ u16 flags; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Size of scalar and vector arguments in bytes. */ u16 scalar_size, vector_size; @@ -140,6 +165,7 @@ typedef struct _vlib_node_registration } vlib_node_registration_t; +#ifndef CLIB_MARCH_VARIANT #define VLIB_REGISTER_NODE(x,...) \ __VA_ARGS__ vlib_node_registration_t x; \ static void __vlib_add_node_registration_##x (void) \ @@ -150,7 +176,46 @@ static void __vlib_add_node_registration_##x (void) \ x.next_registration = vm->node_main.node_registrations; \ vm->node_main.node_registrations = &x; \ } \ +static void __vlib_rm_node_registration_##x (void) \ + __attribute__((__destructor__)) ; \ +static void __vlib_rm_node_registration_##x (void) \ +{ \ + vlib_main_t * vm = vlib_get_main(); \ + VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations, \ + &x, next_registration); \ +} \ __VA_ARGS__ vlib_node_registration_t x +#else +#define VLIB_REGISTER_NODE(x,...) \ +static __clib_unused vlib_node_registration_t __clib_unused_##x +#endif + +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_VARIANT_STR "default" +#else +#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s) +#define __CLIB_MARCH_VARIANT_STR(s) #s +#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT) +#endif + +#define VLIB_NODE_FN(node) \ +uword CLIB_MARCH_SFX (node##_fn)(); \ +static vlib_node_fn_registration_t \ + CLIB_MARCH_SFX(node##_fn_registration) = \ + { .function = &CLIB_MARCH_SFX (node##_fn), }; \ + \ +static void __clib_constructor \ +CLIB_MARCH_SFX (node##_multiarch_register) (void) \ +{ \ + extern vlib_node_registration_t node; \ + vlib_node_fn_registration_t *r; \ + r = & CLIB_MARCH_SFX (node##_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->name = CLIB_MARCH_VARIANT_STR; \ + r->next_registration = node.node_fn_registrations; \ + node.node_fn_registrations = r; \ +} \ +uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn) #if CLIB_DEBUG > 0 #define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) @@ -193,6 +258,8 @@ typedef struct u64 calls, vectors, clocks, suspends; u64 max_clock; u64 max_clock_n; + u64 perf_counter_ticks; + u64 perf_counter_vectors; } vlib_node_stats_t; #define foreach_vlib_node_state \ @@ -267,6 +334,9 @@ typedef struct vlib_node_t /* Number of bytes of run time data. */ u8 runtime_data_bytes; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Number of error codes used by this node. */ u16 n_errors; @@ -320,18 +390,24 @@ typedef struct vlib_node_t struct vlib_frame_t * f); /* for pretty-printing, not typically valid */ u8 *state_string; + + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; } vlib_node_t; #define VLIB_INVALID_NODE_INDEX ((u32) ~0) /* Max number of vector elements to process at once per node. */ #define VLIB_FRAME_SIZE 256 -#define VLIB_FRAME_ALIGN VLIB_MAX_CPUS +#define VLIB_FRAME_ALIGN CLIB_CACHE_LINE_BYTES /* Calling frame (think stack frame) for a node. */ typedef struct vlib_frame_t { /* Frame flags. */ + u16 frame_flags; + + /* User flags. Used for sending hints to the next node. */ u16 flags; /* Number of scalar bytes in arguments. */ @@ -343,9 +419,6 @@ typedef struct vlib_frame_t /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner cpuid / heap id */ - u16 cpu_index; - /* Scalar and vector arguments to next node. */ u8 arguments[0]; } vlib_frame_t; @@ -388,7 +461,7 @@ typedef struct always_inline void vlib_next_frame_init (vlib_next_frame_t * nf) { - memset (nf, 0, sizeof (nf[0])); + clib_memset (nf, 0, sizeof (nf[0])); nf->frame_index = ~0; nf->node_runtime_index = ~0; } @@ -411,65 +484,75 @@ typedef struct typedef struct vlib_node_runtime_t { - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - /* Node function to call. */ - vlib_node_function_t *function; + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /**< cacheline mark */ - /* Vector of errors for this node. */ - vlib_error_t *errors; + vlib_node_function_t *function; /**< Node function to call. */ - /* Number of clock cycles. */ - u32 clocks_since_last_overflow; + vlib_error_t *errors; /**< Vector of errors for this node. */ - /* Maximum clock cycle for an invocation. */ - u32 max_clock; +#if __SIZEOF_POINTER__ == 4 + u8 pad[8]; +#endif - /* Number of vectors in the recorded max_clock. */ - u32 max_clock_n; + u32 clocks_since_last_overflow; /**< Number of clock cycles. */ - /* Number of calls. */ - u32 calls_since_last_overflow; + u32 max_clock; /**< Maximum clock cycle for an + invocation. */ - /* Number of vector elements processed by this node. */ - u32 vectors_since_last_overflow; + u32 max_clock_n; /**< Number of vectors in the recorded + max_clock. */ - /* Start of next frames for this node. */ - u32 next_frame_index; + u32 calls_since_last_overflow; /**< Number of calls. */ - /* Node index. */ - u32 node_index; + u32 vectors_since_last_overflow; /**< Number of vector elements + processed by this node. */ - /* For input nodes: decremented on each main loop interation until it reaches zero - and function is called. Allows some input nodes to be called - more than others. */ - u32 input_main_loops_per_call; + u32 perf_counter_ticks_since_last_overflow; /**< Perf counter ticks */ + u32 perf_counter_vectors_since_last_overflow; /**< Perf counter vectors */ - /* Saved main loop counter of last dispatch of this node. */ - u32 main_loop_count_last_dispatch; + u32 next_frame_index; /**< Start of next frames for this + node. */ - u32 main_loop_vector_stats[2]; + u32 node_index; /**< Node index. */ - /* Copy of main node flags. */ - u16 flags; + u32 input_main_loops_per_call; /**< For input nodes: decremented + on each main loop interation until + it reaches zero and function is + called. Allows some input nodes to + be called more than others. */ - /* Input node state. */ - u16 state; + u32 main_loop_count_last_dispatch; /**< Saved main loop counter of last + dispatch of this node. */ - u16 n_next_nodes; + u32 main_loop_vector_stats[2]; - /* Next frame index that vector arguments were last enqueued to - last time this node ran. Set to zero before first run - of this node. */ - u16 cached_next_index; + u16 flags; /**< Copy of main node flags. */ - /* CPU this node runs on */ - u16 cpu_index; + u16 state; /**< Input node state. */ - /* Function dependent node-runtime. */ - u8 runtime_data[0]; + u16 n_next_nodes; + + u16 cached_next_index; /**< Next frame index that vector + arguments were last enqueued to + last time this node ran. Set to + zero before first run of this + node. */ + + u16 thread_index; /**< thread this node runs on */ + + u8 runtime_data[0]; /**< Function dependent + node-runtime data. This data is + thread local, and it is not + cloned from main thread. It needs + to be initialized for each thread + before it is used unless + runtime_data template exists in + vlib_node_t. */ } vlib_node_runtime_t; +#define VLIB_NODE_RUNTIME_DATA_SIZE (sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)) + typedef struct { /* Number of allocated frames for this scalar/vector size. */ @@ -534,8 +617,14 @@ typedef struct /* Pool of currently valid event types. */ vlib_process_event_type_t *event_type_pool; - /* When suspending saves cpu cycle counter when process is to be resumed. */ - u64 resume_cpu_time; + /* + * When suspending saves clock time (10us ticks) when process + * is to be resumed. + */ + u64 resume_clock_interval; + + /* Handle from timer code, to cancel an unexpired timer */ + u32 stop_timer_handle; /* Default output function and its argument for any CLI outputs within the process. */ @@ -641,6 +730,7 @@ typedef struct /* Node runtime indices for input nodes with pending interrupts. */ u32 *pending_interrupt_node_runtime_indices; + clib_spinlock_t pending_interrupt_lock; /* Input nodes are switched from/to interrupt to/from polling mode when average vector length goes above/below polling/interrupt @@ -655,7 +745,7 @@ typedef struct vlib_pending_frame_t *pending_frames; /* Timing wheel for scheduling time-based node dispatch. */ - timing_wheel_t timing_wheel; + void *timing_wheel; vlib_signal_timed_event_data_t *signal_timed_event_data_pool; @@ -663,7 +753,7 @@ typedef struct u32 *data_from_advancing_timing_wheel; /* CPU time of next process to be ready on timing wheel. */ - u64 cpu_time_next_process_ready; + f64 time_next_process_ready; /* Vector of process nodes. One for each node of type VLIB_NODE_TYPE_PROCESS. */