X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode.h;h=bdc3c6b59ad456f8490baa019859e23dc99d7cde;hb=26e035bfcda580a8ce06df7498ca30e0a39b45ef;hp=2a532cc3988ecbbe2ecd4d416b6d0e447c4b5227;hpb=e9f929b52ddb741ec1e4cb2d92c6be1e798933a0;p=vpp.git diff --git a/src/vlib/node.h b/src/vlib/node.h index 2a532cc3988..bdc3c6b59ad 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -42,7 +42,7 @@ #include #include -#include +#include #include /* for vlib_trace_filter_t */ /* Forward declaration. */ @@ -55,6 +55,17 @@ typedef uword (vlib_node_function_t) (struct vlib_main_t * vm, struct vlib_node_runtime_t * node, struct vlib_frame_t * frame); +typedef enum +{ + VLIB_NODE_PROTO_HINT_NONE = 0, + VLIB_NODE_PROTO_HINT_ETHERNET, + VLIB_NODE_PROTO_HINT_IP4, + VLIB_NODE_PROTO_HINT_IP6, + VLIB_NODE_PROTO_HINT_TCP, + VLIB_NODE_PROTO_HINT_UDP, + VLIB_NODE_N_PROTO_HINTS, +} vlib_node_proto_hint_t; + typedef enum { /* An internal node on the call graph (could be output). */ @@ -75,11 +86,22 @@ typedef enum VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct _vlib_node_fn_registration +{ + vlib_node_function_t *function; + int priority; + struct _vlib_node_fn_registration *next_registration; + char *name; +} vlib_node_fn_registration_t; + typedef struct _vlib_node_registration { /* Vector processing function for this node. */ vlib_node_function_t *function; + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; + /* Node name. */ char *name; @@ -123,6 +145,9 @@ typedef struct _vlib_node_registration /* Node flags. */ u16 flags; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Size of scalar and vector arguments in bytes. */ u16 scalar_size, vector_size; @@ -140,6 +165,7 @@ typedef struct _vlib_node_registration } vlib_node_registration_t; +#ifndef CLIB_MARCH_VARIANT #define VLIB_REGISTER_NODE(x,...) \ __VA_ARGS__ vlib_node_registration_t x; \ static void __vlib_add_node_registration_##x (void) \ @@ -150,34 +176,47 @@ static void __vlib_add_node_registration_##x (void) \ x.next_registration = vm->node_main.node_registrations; \ vm->node_main.node_registrations = &x; \ } \ +static void __vlib_rm_node_registration_##x (void) \ + __attribute__((__destructor__)) ; \ +static void __vlib_rm_node_registration_##x (void) \ +{ \ + vlib_main_t * vm = vlib_get_main(); \ + VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations, \ + &x, next_registration); \ +} \ __VA_ARGS__ vlib_node_registration_t x +#else +#define VLIB_REGISTER_NODE(x,...) \ +static __clib_unused vlib_node_registration_t __clib_unused_##x +#endif -#if CLIB_DEBUG > 0 -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_VARIANT_STR "default" #else -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( struct vlib_main_t * vm, \ - struct vlib_node_runtime_t * node, \ - struct vlib_frame_t * frame) \ - { return fn (vm, node, frame); } - -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn) - -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \ - VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - CLIB_MULTIARCH_SELECT_FN(fn, static inline) \ - static void __attribute__((__constructor__)) \ - __vlib_node_function_multiarch_select_##node (void) \ - { node.function = fn ## _multiarch_select(); } +#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s) +#define __CLIB_MARCH_VARIANT_STR(s) #s +#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT) #endif +#define VLIB_NODE_FN(node) \ +uword CLIB_MARCH_SFX (node##_fn)(); \ +static vlib_node_fn_registration_t \ + CLIB_MARCH_SFX(node##_fn_registration) = \ + { .function = &CLIB_MARCH_SFX (node##_fn), }; \ + \ +static void __clib_constructor \ +CLIB_MARCH_SFX (node##_multiarch_register) (void) \ +{ \ + extern vlib_node_registration_t node; \ + vlib_node_fn_registration_t *r; \ + r = & CLIB_MARCH_SFX (node##_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->name = CLIB_MARCH_VARIANT_STR; \ + r->next_registration = node.node_fn_registrations; \ + node.node_fn_registrations = r; \ +} \ +uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn) + always_inline vlib_node_registration_t * vlib_node_next_registered (vlib_node_registration_t * c) { @@ -193,6 +232,9 @@ typedef struct u64 calls, vectors, clocks, suspends; u64 max_clock; u64 max_clock_n; + u64 perf_counter0_ticks; + u64 perf_counter1_ticks; + u64 perf_counter_vectors; } vlib_node_stats_t; #define foreach_vlib_node_state \ @@ -267,6 +309,9 @@ typedef struct vlib_node_t /* Number of bytes of run time data. */ u8 runtime_data_bytes; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Number of error codes used by this node. */ u16 n_errors; @@ -320,18 +365,24 @@ typedef struct vlib_node_t struct vlib_frame_t * f); /* for pretty-printing, not typically valid */ u8 *state_string; + + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; } vlib_node_t; #define VLIB_INVALID_NODE_INDEX ((u32) ~0) /* Max number of vector elements to process at once per node. */ #define VLIB_FRAME_SIZE 256 -#define VLIB_FRAME_ALIGN VLIB_MAX_CPUS +#define VLIB_FRAME_ALIGN CLIB_CACHE_LINE_BYTES /* Calling frame (think stack frame) for a node. */ typedef struct vlib_frame_t { /* Frame flags. */ + u16 frame_flags; + + /* User flags. Used for sending hints to the next node. */ u16 flags; /* Number of scalar bytes in arguments. */ @@ -343,9 +394,6 @@ typedef struct vlib_frame_t /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner cpuid / heap id */ - u16 cpu_index; - /* Scalar and vector arguments to next node. */ u8 arguments[0]; } vlib_frame_t; @@ -365,6 +413,9 @@ typedef struct #define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \ VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH + /* Don't append this frame */ +#define VLIB_FRAME_NO_APPEND (1 << 14) + /* This next frame owns enqueue to node corresponding to node_runtime_index. */ #define VLIB_FRAME_OWNER (1 << 15) @@ -388,7 +439,7 @@ typedef struct always_inline void vlib_next_frame_init (vlib_next_frame_t * nf) { - memset (nf, 0, sizeof (nf[0])); + clib_memset (nf, 0, sizeof (nf[0])); nf->frame_index = ~0; nf->node_runtime_index = ~0; } @@ -417,6 +468,10 @@ typedef struct vlib_node_runtime_t vlib_error_t *errors; /**< Vector of errors for this node. */ +#if __SIZEOF_POINTER__ == 4 + u8 pad[8]; +#endif + u32 clocks_since_last_overflow; /**< Number of clock cycles. */ u32 max_clock; /**< Maximum clock cycle for an @@ -430,6 +485,10 @@ typedef struct vlib_node_runtime_t u32 vectors_since_last_overflow; /**< Number of vector elements processed by this node. */ + u32 perf_counter0_ticks_since_last_overflow; /**< Perf counter 0 ticks */ + u32 perf_counter1_ticks_since_last_overflow; /**< Perf counter 1 ticks */ + u32 perf_counter_vectors_since_last_overflow; /**< Perf counter vectors */ + u32 next_frame_index; /**< Start of next frames for this node. */ @@ -458,7 +517,7 @@ typedef struct vlib_node_runtime_t zero before first run of this node. */ - u16 cpu_index; /**< CPU this node runs on */ + u16 thread_index; /**< thread this node runs on */ u8 runtime_data[0]; /**< Function dependent node-runtime data. This data is @@ -537,8 +596,14 @@ typedef struct /* Pool of currently valid event types. */ vlib_process_event_type_t *event_type_pool; - /* When suspending saves cpu cycle counter when process is to be resumed. */ - u64 resume_cpu_time; + /* + * When suspending saves clock time (10us ticks) when process + * is to be resumed. + */ + u64 resume_clock_interval; + + /* Handle from timer code, to cancel an unexpired timer */ + u32 stop_timer_handle; /* Default output function and its argument for any CLI outputs within the process. */ @@ -644,6 +709,7 @@ typedef struct /* Node runtime indices for input nodes with pending interrupts. */ u32 *pending_interrupt_node_runtime_indices; + clib_spinlock_t pending_interrupt_lock; /* Input nodes are switched from/to interrupt to/from polling mode when average vector length goes above/below polling/interrupt @@ -658,7 +724,7 @@ typedef struct vlib_pending_frame_t *pending_frames; /* Timing wheel for scheduling time-based node dispatch. */ - timing_wheel_t timing_wheel; + void *timing_wheel; vlib_signal_timed_event_data_t *signal_timed_event_data_pool; @@ -666,7 +732,7 @@ typedef struct u32 *data_from_advancing_timing_wheel; /* CPU time of next process to be ready on timing wheel. */ - u64 cpu_time_next_process_ready; + f64 time_next_process_ready; /* Vector of process nodes. One for each node of type VLIB_NODE_TYPE_PROCESS. */