X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode.h;h=68813c2c3e1993a8766e25b3e872dd3fa1f3df86;hb=HEAD;hp=906d795fe239a18946731e8769f9fdf2f4ce2a4f;hpb=36c1308b35baac817c2c0b21335b1eed02689f0a;p=vpp.git diff --git a/src/vlib/node.h b/src/vlib/node.h index 906d795fe23..68813c2c3e1 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -43,7 +43,6 @@ #include #include #include -#include #include /* for vlib_trace_filter_t */ /* Forward declaration. */ @@ -56,6 +55,17 @@ typedef uword (vlib_node_function_t) (struct vlib_main_t * vm, struct vlib_node_runtime_t * node, struct vlib_frame_t * frame); +typedef enum +{ + VLIB_NODE_PROTO_HINT_NONE = 0, + VLIB_NODE_PROTO_HINT_ETHERNET, + VLIB_NODE_PROTO_HINT_IP4, + VLIB_NODE_PROTO_HINT_IP6, + VLIB_NODE_PROTO_HINT_TCP, + VLIB_NODE_PROTO_HINT_UDP, + VLIB_NODE_N_PROTO_HINTS, +} vlib_node_proto_hint_t; + typedef enum { /* An internal node on the call graph (could be output). */ @@ -76,11 +86,21 @@ typedef enum VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct _vlib_node_fn_registration +{ + vlib_node_function_t *function; + clib_march_variant_type_t march_variant; + struct _vlib_node_fn_registration *next_registration; +} vlib_node_fn_registration_t; + typedef struct _vlib_node_registration { /* Vector processing function for this node. */ vlib_node_function_t *function; + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; + /* Node name. */ char *name; @@ -95,6 +115,7 @@ typedef struct _vlib_node_registration /* Error strings indexed by error code for this node. */ char **error_strings; + vlib_error_desc_t *error_counters; /* Buffer format/unformat for this node. */ format_function_t *format_buffer; @@ -124,8 +145,12 @@ typedef struct _vlib_node_registration /* Node flags. */ u16 flags; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Size of scalar and vector arguments in bytes. */ - u16 scalar_size, vector_size; + u8 vector_size, aux_size; + u16 scalar_size; /* Number of error codes used by this node. */ u16 n_errors; @@ -141,52 +166,61 @@ typedef struct _vlib_node_registration } vlib_node_registration_t; +#ifndef CLIB_MARCH_VARIANT +#define VLIB_REGISTER_NODE(x, ...) \ + __VA_ARGS__ vlib_node_registration_t x; \ + static void __vlib_add_node_registration_##x (void) \ + __attribute__ ((__constructor__)); \ + static void __vlib_add_node_registration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + x.next_registration = vgm->node_registrations; \ + vgm->node_registrations = &x; \ + } \ + static void __vlib_rm_node_registration_##x (void) \ + __attribute__ ((__destructor__)); \ + static void __vlib_rm_node_registration_##x (void) \ + { \ + vlib_global_main_t *vgm = vlib_get_global_main (); \ + VLIB_REMOVE_FROM_LINKED_LIST (vgm->node_registrations, &x, \ + next_registration); \ + } \ + __VA_ARGS__ vlib_node_registration_t x +#else #define VLIB_REGISTER_NODE(x,...) \ - __VA_ARGS__ vlib_node_registration_t x; \ -static void __vlib_add_node_registration_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_add_node_registration_##x (void) \ -{ \ - vlib_main_t * vm = vlib_get_main(); \ - x.next_registration = vm->node_main.node_registrations; \ - vm->node_main.node_registrations = &x; \ -} \ -__VA_ARGS__ vlib_node_registration_t x - -#if CLIB_DEBUG > 0 -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) +STATIC_ASSERT (sizeof(# __VA_ARGS__) != 7,"node " #x " must not be declared as static"); \ +static __clib_unused vlib_node_registration_t __clib_unused_##x +#endif + +#ifndef CLIB_MARCH_VARIANT +#define CLIB_MARCH_VARIANT_STR "default" #else -#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \ - uword \ - __attribute__ ((flatten)) \ - __attribute__ ((target (tgt))) \ - CLIB_CPU_OPTIMIZED \ - fn ## _ ## arch ( struct vlib_main_t * vm, \ - struct vlib_node_runtime_t * node, \ - struct vlib_frame_t * frame) \ - { return fn (vm, node, frame); } - -#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn) - -#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \ - VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ - CLIB_MULTIARCH_SELECT_FN(fn, static inline) \ - static void __attribute__((__constructor__)) \ - __vlib_node_function_multiarch_select_##node (void) \ - { node.function = fn ## _multiarch_select(); } +#define _CLIB_MARCH_VARIANT_STR(s) __CLIB_MARCH_VARIANT_STR(s) +#define __CLIB_MARCH_VARIANT_STR(s) #s +#define CLIB_MARCH_VARIANT_STR _CLIB_MARCH_VARIANT_STR(CLIB_MARCH_VARIANT) #endif -always_inline vlib_node_registration_t * -vlib_node_next_registered (vlib_node_registration_t * c) -{ - c = - clib_elf_section_data_next (c, - c->n_next_nodes * sizeof (c->next_nodes[0])); - return c; -} +#define VLIB_NODE_FN(node) \ + uword CLIB_MARCH_SFX (node##_fn) (vlib_main_t *, vlib_node_runtime_t *, \ + vlib_frame_t *); \ + static vlib_node_fn_registration_t CLIB_MARCH_SFX ( \ + node##_fn_registration) = { \ + .function = &CLIB_MARCH_SFX (node##_fn), \ + }; \ + \ + static void __clib_constructor CLIB_MARCH_SFX (node##_multiarch_register) ( \ + void) \ + { \ + extern vlib_node_registration_t node; \ + vlib_node_fn_registration_t *r; \ + r = &CLIB_MARCH_SFX (node##_fn_registration); \ + r->march_variant = CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE); \ + r->next_registration = node.node_fn_registrations; \ + node.node_fn_registrations = r; \ + } \ + uword CLIB_MARCH_SFX (node##_fn) + +unformat_function_t unformat_vlib_node_variant; typedef struct { @@ -241,7 +275,7 @@ typedef struct vlib_node_t u32 runtime_index; /* Runtime data for this node. */ - void *runtime_data; + u8 *runtime_data; /* Node flags. */ u16 flags; @@ -261,6 +295,8 @@ typedef struct vlib_node_t #define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE (1 << 6) #define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE (1 << 7) +#define VLIB_NODE_FLAG_TRACE_SUPPORTED (1 << 8) +#define VLIB_NODE_FLAG_ADAPTIVE_MODE (1 << 9) /* State for input nodes. */ u8 state; @@ -268,18 +304,22 @@ typedef struct vlib_node_t /* Number of bytes of run time data. */ u8 runtime_data_bytes; + /* protocol at b->data[b->current_data] upon entry to the dispatch fn */ + u8 protocol_hint; + /* Number of error codes used by this node. */ u16 n_errors; /* Size of scalar and vector arguments in bytes. */ - u16 scalar_size, vector_size; + u16 frame_size, scalar_offset, vector_offset, magic_offset, aux_offset; + u16 frame_size_index; /* Handle/index in error heap for this node. */ u32 error_heap_handle; u32 error_heap_index; - /* Error strings indexed by error code for this node. */ - char **error_strings; + /* Counter structures indexed by counter code for this node. */ + vlib_error_desc_t *error_counters; /* Vector of next node names. Only used before next_nodes array is initialized. */ @@ -321,31 +361,37 @@ typedef struct vlib_node_t struct vlib_frame_t * f); /* for pretty-printing, not typically valid */ u8 *state_string; + + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; } vlib_node_t; #define VLIB_INVALID_NODE_INDEX ((u32) ~0) /* Max number of vector elements to process at once per node. */ #define VLIB_FRAME_SIZE 256 -#define VLIB_FRAME_ALIGN VLIB_MAX_CPUS +/* Number of extra elements allocated at the end of vecttor. */ +#define VLIB_FRAME_SIZE_EXTRA 4 +/* Frame data alignment */ +#define VLIB_FRAME_DATA_ALIGN 16 /* Calling frame (think stack frame) for a node. */ typedef struct vlib_frame_t { /* Frame flags. */ - u16 flags; + u16 frame_flags; - /* Number of scalar bytes in arguments. */ - u8 scalar_size; + /* User flags. Used for sending hints to the next node. */ + u16 flags; - /* Number of bytes per vector argument. */ - u8 vector_size; + /* Scalar, vector and aux offsets in this frame. */ + u16 scalar_offset, vector_offset, aux_offset; /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner thread / heap id */ - u16 thread_index; + /* Index of frame size corresponding to allocated node. */ + u16 frame_size_index; /* Scalar and vector arguments to next node. */ u8 arguments[0]; @@ -353,8 +399,8 @@ typedef struct vlib_frame_t typedef struct { - /* Frame index. */ - u32 frame_index; + /* Frame pointer. */ + vlib_frame_t *frame; /* Node runtime for this next. */ u32 node_runtime_index; @@ -366,6 +412,9 @@ typedef struct #define VLIB_FRAME_NO_FREE_AFTER_DISPATCH \ VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH + /* Don't append this frame */ +#define VLIB_FRAME_NO_APPEND (1 << 14) + /* This next frame owns enqueue to node corresponding to node_runtime_index. */ #define VLIB_FRAME_OWNER (1 << 15) @@ -389,20 +438,19 @@ typedef struct always_inline void vlib_next_frame_init (vlib_next_frame_t * nf) { - memset (nf, 0, sizeof (nf[0])); - nf->frame_index = ~0; + clib_memset (nf, 0, sizeof (nf[0])); nf->node_runtime_index = ~0; } /* A frame pending dispatch by main loop. */ typedef struct { + /* Frame index (in the heap). */ + vlib_frame_t *frame; + /* Node and runtime for this frame. */ u32 node_runtime_index; - /* Frame index (in the heap). */ - u32 frame_index; - /* Start of next frames for this node. */ u32 next_frame_index; @@ -418,10 +466,6 @@ typedef struct vlib_node_runtime_t vlib_error_t *errors; /**< Vector of errors for this node. */ -#if __SIZEOF_POINTER__ == 4 - u8 pad[8]; -#endif - u32 clocks_since_last_overflow; /**< Number of clock cycles. */ u32 max_clock; /**< Maximum clock cycle for an @@ -463,7 +507,7 @@ typedef struct vlib_node_runtime_t zero before first run of this node. */ - u16 thread_index; /**< thread this node runs on */ + CLIB_ALIGN_MARK (runtime_data_pad, 8); u8 runtime_data[0]; /**< Function dependent node-runtime data. This data is @@ -483,10 +527,15 @@ typedef struct /* Number of allocated frames for this scalar/vector size. */ u32 n_alloc_frames; - /* Vector of free frame indices for this scalar/vector size. */ - u32 *free_frame_indices; + /* Frame size */ + u16 frame_size; + + /* Vector of free frames for this scalar/vector size. */ + vlib_frame_t **free_frames; } vlib_frame_size_t; +STATIC_ASSERT_SIZEOF (vlib_frame_size_t, 16); + typedef struct { /* Users opaque value for event type. */ @@ -495,6 +544,7 @@ typedef struct typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* Node runtime for this process. */ vlib_node_runtime_t node_runtime; @@ -527,7 +577,7 @@ typedef struct u32 n_suspends; /* Vectors of pending event data indexed by event type index. */ - void **pending_event_data_by_type_index; + u8 **pending_event_data_by_type_index; /* Bitmap of event type-indices with non-empty vectors. */ uword *non_empty_event_type_bitmap; @@ -542,40 +592,24 @@ typedef struct /* Pool of currently valid event types. */ vlib_process_event_type_t *event_type_pool; - /* When suspending saves cpu cycle counter when process is to be resumed. */ - u64 resume_cpu_time; + /* + * When suspending saves clock time (10us ticks) when process + * is to be resumed. + */ + u64 resume_clock_interval; + + /* Handle from timer code, to cancel an unexpired timer */ + u32 stop_timer_handle; /* Default output function and its argument for any CLI outputs within the process. */ vlib_cli_output_function_t *output_function; uword output_function_arg; -#ifdef CLIB_UNIX - /* Pad to a multiple of the page size so we can mprotect process stacks */ -#define PAGE_SIZE_MULTIPLE 0x1000 -#define ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT __attribute__ ((aligned (PAGE_SIZE_MULTIPLE))) -#else -#define ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT -#endif - - /* Process stack. Starts here and extends 2^log2_n_stack_bytes - bytes. */ - + /* Process stack */ #define VLIB_PROCESS_STACK_MAGIC (0xdead7ead) - u32 stack[0] ALIGN_ON_MULTIPLE_PAGE_BOUNDARY_FOR_MPROTECT; -} vlib_process_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))); - -#ifdef CLIB_UNIX - /* Ensure that the stack is aligned on the multiple of the page size */ -typedef char - assert_process_stack_must_be_aligned_exactly_to_page_size_multiple[(sizeof - (vlib_process_t) - - - PAGE_SIZE_MULTIPLE) - == - 0 ? 0 : - -1]; -#endif + u32 *stack; +} vlib_process_t; typedef struct { @@ -632,6 +666,14 @@ vlib_timing_wheel_data_get_index (u32 d) return d / 2; } +typedef struct +{ + clib_march_variant_type_t index; + int priority; + char *suffix; + char *desc; +} vlib_node_fn_variant_t; + typedef struct { /* Public nodes. */ @@ -648,8 +690,8 @@ typedef struct vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE]; /* Node runtime indices for input nodes with pending interrupts. */ - u32 *pending_interrupt_node_runtime_indices; - clib_spinlock_t pending_interrupt_lock; + void *input_node_interrupts; + void *pre_input_node_interrupts; /* Input nodes are switched from/to interrupt to/from polling mode when average vector length goes above/below polling/interrupt @@ -664,7 +706,7 @@ typedef struct vlib_pending_frame_t *pending_frames; /* Timing wheel for scheduling time-based node dispatch. */ - timing_wheel_t timing_wheel; + void *timing_wheel; vlib_signal_timed_event_data_t *signal_timed_event_data_pool; @@ -672,7 +714,7 @@ typedef struct u32 *data_from_advancing_timing_wheel; /* CPU time of next process to be ready on timing wheel. */ - u64 cpu_time_next_process_ready; + f64 time_next_process_ready; /* Vector of process nodes. One for each node of type VLIB_NODE_TYPE_PROCESS. */ @@ -690,26 +732,47 @@ typedef struct /* Current counts of nodes in each state. */ u32 input_node_counts_by_state[VLIB_N_NODE_STATE]; - /* Hash of (scalar_size,vector_size) to frame_sizes index. */ - uword *frame_size_hash; - /* Per-size frame allocation information. */ vlib_frame_size_t *frame_sizes; /* Time of last node runtime stats clear. */ f64 time_last_runtime_stats_clear; - /* Node registrations added by constructors */ - vlib_node_registration_t *node_registrations; + /* Node index from error code */ + u32 *node_by_error; + + /* Node Function Variants */ + vlib_node_fn_variant_t *variants; + + /* Node Function Default Variant Index */ + u32 node_fn_default_march_variant; + + /* Node Function march Variant by Suffix Hash */ + uword *node_fn_march_variant_by_suffix; } vlib_node_main_t; +typedef u16 vlib_error_t; + +always_inline u32 +vlib_error_get_node (vlib_node_main_t * nm, vlib_error_t e) +{ + return nm->node_by_error[e]; +} + +always_inline u32 +vlib_error_get_code (vlib_node_main_t * nm, vlib_error_t e) +{ + u32 node_index = nm->node_by_error[e]; + vlib_node_t *n = nm->nodes[node_index]; + u32 error_code = e - n->error_heap_index; + return error_code; +} -#define FRAME_QUEUE_MAX_NELTS 32 +#define FRAME_QUEUE_MAX_NELTS 64 typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u64 head; - u64 head_hint; u64 tail; u32 n_in_use; u32 nelts;