X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode.h;h=b9961f55b562bfd97b0112f3817f3b5aa1440bd4;hb=bdfe5955f;hp=fc7e7da2b8598ce8aa11691c1f16635ba675ec12;hpb=2c2b6407129461e183bc8ad5964cddc98ed1626c;p=vpp.git diff --git a/src/vlib/node.h b/src/vlib/node.h index fc7e7da2b85..b9961f55b56 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -43,7 +43,6 @@ #include #include #include -#include #include /* for vlib_trace_filter_t */ /* Forward declaration. */ @@ -56,6 +55,17 @@ typedef uword (vlib_node_function_t) (struct vlib_main_t * vm, struct vlib_node_runtime_t * node, struct vlib_frame_t * frame); +typedef enum +{ + VLIB_NODE_PROTO_HINT_NONE = 0, + VLIB_NODE_PROTO_HINT_ETHERNET, + VLIB_NODE_PROTO_HINT_IP4, + VLIB_NODE_PROTO_HINT_IP6, + VLIB_NODE_PROTO_HINT_TCP, + VLIB_NODE_PROTO_HINT_UDP, + VLIB_NODE_N_PROTO_HINTS, +} vlib_node_proto_hint_t; + typedef enum { /* An internal node on the call graph (could be output). */ @@ -76,11 +86,22 @@ typedef enum VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct _vlib_node_fn_registration +{ + vlib_node_function_t *function; + int priority; + struct _vlib_node_fn_registration *next_registration; + char *name; +} vlib_node_fn_registration_t; + typedef struct _vlib_node_registration { /* Vector processing function for this node. */ vlib_node_function_t *function; + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; + /* Node name. */ char *name; @@ -124,6 +145,9 @@ typedef struct _vlib_node_registration /* Node flags. */ u16 flags; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Size of scalar and vector arguments in bytes. */ u16 scalar_size, vector_size; @@ -141,6 +165,7 @@ typedef struct _vlib_node_registration } vlib_node_registration_t; +#ifndef CLIB_MARCH_VARIANT #define VLIB_REGISTER_NODE(x,...) \ __VA_ARGS__ vlib_node_registration_t x; \ static void __vlib_add_node_registration_##x (void) \ @@ -151,34 +176,50 @@ static void __vlib_add_node_registration_##x (void) \ x.next_registration = vm->node_main.node_registrations; \ vm->node_main.node_registrations = &x; \ } \ +static void __vlib_rm_node_registration_##x (void) \ + __attribute__((__destructor__)) ; \ +static void __vlib_rm_node_registration_##x (void) \ +{ \ + vlib_main_t * vm = vlib_get_main(); \ + VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations, \ + &x, next_registration); \ +} \ __VA_ARGS__ vlib_node_registration_t x +#else +#define VLIB_REGISTER_NODE(x,...) \ +STATIC_ASSERT (sizeof(# __VA_ARGS__) != 7,"node " #x " must not be declared as static"); \ +static __clib_unused vlib_node_registration_t __clib_unused_##x +#endif -#if CLIB_DEBUG > 0 -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_VARIANT_STR "default" #else -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( struct vlib_main_t * vm, \ - struct vlib_node_runtime_t * node, \ - struct vlib_frame_t * frame) \ - { return fn (vm, node, frame); } - -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn) - -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \ - VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - CLIB_MULTIARCH_SELECT_FN(fn, static inline) \ - static void __attribute__((__constructor__)) \ - __vlib_node_function_multiarch_select_##node (void) \ - { node.function = fn ## _multiarch_select(); } +#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s) +#define __CLIB_MARCH_VARIANT_STR(s) #s +#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT) #endif +#define VLIB_NODE_FN(node) \ +uword CLIB_MARCH_SFX (node##_fn)(); \ +static vlib_node_fn_registration_t \ + CLIB_MARCH_SFX(node##_fn_registration) = \ + { .function = &CLIB_MARCH_SFX (node##_fn), }; \ + \ +static void __clib_constructor \ +CLIB_MARCH_SFX (node##_multiarch_register) (void) \ +{ \ + extern vlib_node_registration_t node; \ + vlib_node_fn_registration_t *r; \ + r = & CLIB_MARCH_SFX (node##_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->name = CLIB_MARCH_VARIANT_STR; \ + r->next_registration = node.node_fn_registrations; \ + node.node_fn_registrations = r; \ +} \ +uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn) + +unformat_function_t unformat_vlib_node_variant; + always_inline vlib_node_registration_t * vlib_node_next_registered (vlib_node_registration_t * c) { @@ -194,6 +235,9 @@ typedef struct u64 calls, vectors, clocks, suspends; u64 max_clock; u64 max_clock_n; + u64 perf_counter0_ticks; + u64 perf_counter1_ticks; + u64 perf_counter_vectors; } vlib_node_stats_t; #define foreach_vlib_node_state \ @@ -261,6 +305,7 @@ typedef struct vlib_node_t #define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE (1 << 6) #define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE (1 << 7) +#define VLIB_NODE_FLAG_TRACE_SUPPORTED (1 << 8) /* State for input nodes. */ u8 state; @@ -268,6 +313,9 @@ typedef struct vlib_node_t /* Number of bytes of run time data. */ u8 runtime_data_bytes; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Number of error codes used by this node. */ u16 n_errors; @@ -321,18 +369,24 @@ typedef struct vlib_node_t struct vlib_frame_t * f); /* for pretty-printing, not typically valid */ u8 *state_string; + + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; } vlib_node_t; #define VLIB_INVALID_NODE_INDEX ((u32) ~0) /* Max number of vector elements to process at once per node. */ #define VLIB_FRAME_SIZE 256 -#define VLIB_FRAME_ALIGN VLIB_MAX_CPUS +#define VLIB_FRAME_ALIGN CLIB_CACHE_LINE_BYTES /* Calling frame (think stack frame) for a node. */ typedef struct vlib_frame_t { /* Frame flags. */ + u16 frame_flags; + + /* User flags. Used for sending hints to the next node. */ u16 flags; /* Number of scalar bytes in arguments. */ @@ -344,17 +398,14 @@ typedef struct vlib_frame_t /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner cpuid / heap id */ - u16 cpu_index; - /* Scalar and vector arguments to next node. */ u8 arguments[0]; } vlib_frame_t; typedef struct { - /* Frame index. */ - u32 frame_index; + /* Frame pointer. */ + vlib_frame_t *frame; /* Node runtime for this next. */ u32 node_runtime_index; @@ -366,6 +417,9 @@ typedef struct #define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \ VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH + /* Don't append this frame */ +#define VLIB_FRAME_NO_APPEND (1 << 14) + /* This next frame owns enqueue to node corresponding to node_runtime_index. */ #define VLIB_FRAME_OWNER (1 << 15) @@ -389,8 +443,7 @@ typedef struct always_inline void vlib_next_frame_init (vlib_next_frame_t * nf) { - memset (nf, 0, sizeof (nf[0])); - nf->frame_index = ~0; + clib_memset (nf, 0, sizeof (nf[0])); nf->node_runtime_index = ~0; } @@ -401,7 +454,7 @@ typedef struct u32 node_runtime_index; /* Frame index (in the heap). */ - u32 frame_index; + vlib_frame_t *frame; /* Start of next frames for this node. */ u32 next_frame_index; @@ -418,6 +471,10 @@ typedef struct vlib_node_runtime_t vlib_error_t *errors; /**< Vector of errors for this node. */ +#if __SIZEOF_POINTER__ == 4 + u8 pad[8]; +#endif + u32 clocks_since_last_overflow; /**< Number of clock cycles. */ u32 max_clock; /**< Maximum clock cycle for an @@ -431,6 +488,10 @@ typedef struct vlib_node_runtime_t u32 vectors_since_last_overflow; /**< Number of vector elements processed by this node. */ + u32 perf_counter0_ticks_since_last_overflow; /**< Perf counter 0 ticks */ + u32 perf_counter1_ticks_since_last_overflow; /**< Perf counter 1 ticks */ + u32 perf_counter_vectors_since_last_overflow; /**< Perf counter vectors */ + u32 next_frame_index; /**< Start of next frames for this node. */ @@ -459,7 +520,7 @@ typedef struct vlib_node_runtime_t zero before first run of this node. */ - u16 cpu_index; /**< CPU this node runs on */ + u16 thread_index; /**< thread this node runs on */ u8 runtime_data[0]; /**< Function dependent node-runtime data. This data is @@ -479,8 +540,8 @@ typedef struct /* Number of allocated frames for this scalar/vector size. */ u32 n_alloc_frames; - /* Vector of free frame indices for this scalar/vector size. */ - u32 *free_frame_indices; + /* Vector of free frames for this scalar/vector size. */ + vlib_frame_t **free_frames; } vlib_frame_size_t; typedef struct @@ -538,8 +599,14 @@ typedef struct /* Pool of currently valid event types. */ vlib_process_event_type_t *event_type_pool; - /* When suspending saves cpu cycle counter when process is to be resumed. */ - u64 resume_cpu_time; + /* + * When suspending saves clock time (10us ticks) when process + * is to be resumed. + */ + u64 resume_clock_interval; + + /* Handle from timer code, to cancel an unexpired timer */ + u32 stop_timer_handle; /* Default output function and its argument for any CLI outputs within the process. */ @@ -660,7 +727,7 @@ typedef struct vlib_pending_frame_t *pending_frames; /* Timing wheel for scheduling time-based node dispatch. */ - timing_wheel_t timing_wheel; + void *timing_wheel; vlib_signal_timed_event_data_t *signal_timed_event_data_pool; @@ -668,7 +735,7 @@ typedef struct u32 *data_from_advancing_timing_wheel; /* CPU time of next process to be ready on timing wheel. */ - u64 cpu_time_next_process_ready; + f64 time_next_process_ready; /* Vector of process nodes. One for each node of type VLIB_NODE_TYPE_PROCESS. */ @@ -697,10 +764,29 @@ typedef struct /* Node registrations added by constructors */ vlib_node_registration_t *node_registrations; + + /* Node index from error code */ + u32 *node_by_error; } vlib_node_main_t; +typedef u16 vlib_error_t; + +always_inline u32 +vlib_error_get_node (vlib_node_main_t * nm, vlib_error_t e) +{ + return nm->node_by_error[e]; +} + +always_inline u32 +vlib_error_get_code (vlib_node_main_t * nm, vlib_error_t e) +{ + u32 node_index = nm->node_by_error[e]; + vlib_node_t *n = nm->nodes[node_index]; + u32 error_code = e - n->error_heap_index; + return error_code; +} -#define FRAME_QUEUE_MAX_NELTS 32 +#define FRAME_QUEUE_MAX_NELTS 64 typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);