X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fmain.h;h=c655560d08ce6188a2846758dcfb799a5e852021;hb=06111a837;hp=4192c3f27208acc2654e1f495fadb4b75953b5b4;hpb=a4324a996f34902579338033110d27575a654c8e;p=vpp.git diff --git a/src/vlib/main.h b/src/vlib/main.h index 4192c3f2720..c655560d08c 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -40,14 +40,13 @@ #ifndef included_vlib_main_h #define included_vlib_main_h +#include #include #include #include #include #include #include -#include -#include #include @@ -58,6 +57,48 @@ #define VLIB_ELOG_MAIN_LOOP 0 #endif +typedef struct +{ + u8 trace_filter_enable; + u32 classify_table_index; +} vlib_trace_filter_t; + +typedef enum +{ + VLIB_NODE_RUNTIME_PERF_BEFORE, + VLIB_NODE_RUNTIME_PERF_AFTER, + VLIB_NODE_RUNTIME_PERF_RESET, +} vlib_node_runtime_perf_call_type_t; + +typedef struct +{ + struct vlib_main_t *vm; + vlib_node_runtime_t *node; + vlib_frame_t *frame; + uword packets; + u64 cpu_time_now; + vlib_node_runtime_perf_call_type_t call_type; +} vlib_node_runtime_perf_callback_args_t; + +struct vlib_node_runtime_perf_callback_data_t; + +typedef void (*vlib_node_runtime_perf_callback_fp_t) + (struct vlib_node_runtime_perf_callback_data_t * data, + vlib_node_runtime_perf_callback_args_t * args); + +typedef struct vlib_node_runtime_perf_callback_data_t +{ + vlib_node_runtime_perf_callback_fp_t fp; + union + { + void *v; + u64 u; + } u[3]; +} vlib_node_runtime_perf_callback_data_t; + +clib_callback_data_typedef (vlib_node_runtime_perf_callback_set_t, + vlib_node_runtime_perf_callback_data_t); + typedef struct vlib_main_t { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); @@ -74,21 +115,26 @@ typedef struct vlib_main_t u64 cpu_time_main_loop_start; /* Incremented once for each main loop. */ - u32 main_loop_count; + volatile u32 main_loop_count; /* Count of vectors processed this main loop. */ u32 main_loop_vectors_processed; u32 main_loop_nodes_processed; - /* Circular buffer of input node vector counts. - Indexed by low bits of - (main_loop_count >> VLIB_LOG2_INPUT_VECTORS_PER_MAIN_LOOP). */ - u32 vector_counts_per_main_loop[2]; - u32 node_counts_per_main_loop[2]; + /* Internal node vectors, calls */ + u64 internal_node_vectors; + u64 internal_node_calls; + u64 internal_node_vectors_last_clear; + u64 internal_node_calls_last_clear; + + /* Instantaneous vector rate */ + u32 internal_node_last_vectors_per_main_loop; /* Main loop hw / sw performance counters */ - void (*vlib_node_runtime_perf_counter_cb) (struct vlib_main_t *, - u64 *, u64 *); + vlib_node_runtime_perf_callback_set_t vlib_node_runtime_perf_callbacks; + + /* dispatch wrapper function */ + vlib_node_function_t *dispatch_wrapper_fn; /* Every so often we switch to the next counter. */ #define VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE 7 @@ -106,9 +152,6 @@ typedef struct vlib_main_t /* Error marker to use when exiting main loop. */ clib_error_t *main_loop_error; - /* Name for e.g. syslog. */ - char *name; - /* Start of the heap. */ void *heap_base; @@ -127,17 +170,9 @@ typedef struct vlib_main_t /* Node graph main structure. */ vlib_node_main_t node_main; - /* Command line interface. */ - vlib_cli_main_t cli_main; - /* Packet trace buffer. */ vlib_trace_main_t trace_main; - /* Pcap dispatch trace main */ - pcap_main_t dispatch_pcap_main; - uword dispatch_pcap_enable; - u8 *pcap_buffer; - /* Error handling. */ vlib_error_main_t error_main; @@ -150,10 +185,10 @@ typedef struct vlib_main_t /* Stream index to use for distribution when MC is enabled. */ u32 mc_stream_index; - vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join; + /* Hash table to record which init functions have been called. */ + uword *worker_init_functions_called; - /* Event logger. */ - elog_main_t elog_main; + vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join; /* Event logger trace flags */ int elog_trace_api_messages; @@ -174,22 +209,11 @@ typedef struct vlib_main_t /* Buffer of random data for various uses. */ clib_random_buffer_t random_buffer; - /* Hash table to record which init functions have been called. */ - uword *init_functions_called; - /* thread, cpu and numa_node indices */ u32 thread_index; u32 cpu_id; u32 numa_node; - /* List of init functions to call, setup by constructors */ - _vlib_init_function_list_elt_t *init_function_registrations; - _vlib_init_function_list_elt_t *worker_init_function_registrations; - _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; - _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; - _vlib_init_function_list_elt_t *api_init_function_registrations; - vlib_config_function_runtime_t *config_function_registrations; - /* control-plane API queue signal pending, length indication */ volatile u32 queue_signal_pending; volatile u32 api_queue_nonempty; @@ -197,19 +221,22 @@ typedef struct vlib_main_t u8 **argv; /* Top of (worker) dispatch loop callback */ - volatile void (*worker_thread_main_loop_callback) (struct vlib_main_t *); + void (**volatile worker_thread_main_loop_callbacks) + (struct vlib_main_t *, u64 t); + void (**volatile worker_thread_main_loop_callback_tmp) + (struct vlib_main_t *, u64 t); + clib_spinlock_t worker_thread_main_loop_callback_lock; /* debugging */ volatile int parked_at_barrier; - /* Attempt to do a post-mortem elog dump */ - int elog_post_mortem_dump; - - /* - * Need to call vlib_worker_thread_node_runtime_update before - * releasing worker thread barrier. Only valid in vlib_global_main. - */ - int need_vlib_worker_thread_node_runtime_update; + /* Dispatch loop time accounting */ + u64 loops_this_reporting_interval; + f64 loop_interval_end; + f64 loop_interval_start; + f64 loops_per_second; + f64 seconds_per_loop; + f64 damping_constant; /* * Barrier epoch - Set to current time, each time barrier_sync or @@ -220,21 +247,90 @@ typedef struct vlib_main_t /* Earliest barrier can be closed again */ f64 barrier_no_close_before; + /* Barrier counter callback */ + void (**volatile barrier_perf_callbacks) + (struct vlib_main_t *, u64 t, int leave); + void (**volatile barrier_perf_callbacks_tmp) + (struct vlib_main_t *, u64 t, int leave); + + /* Need to check the frame queues */ + volatile uword check_frame_queues; + /* RPC requests, main thread only */ uword *pending_rpc_requests; uword *processing_rpc_requests; clib_spinlock_t pending_rpc_lock; + /* buffer fault injector */ + u32 buffer_alloc_success_seed; + f64 buffer_alloc_success_rate; + +#ifdef CLIB_SANITIZE_ADDR + /* address sanitizer stack save */ + void *asan_stack_save; +#endif } vlib_main_t; +typedef struct vlib_global_main_t +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + + /* Per-thread Mains */ + vlib_main_t **vlib_mains; + + /* Name for e.g. syslog. */ + char *name; + + /* post-mortem callbacks */ + void (**post_mortem_callbacks) (void); + + /* + * Need to call vlib_worker_thread_node_runtime_update before + * releasing worker thread barrier. + */ + int need_vlib_worker_thread_node_runtime_update; + + /* Command line interface. */ + vlib_cli_main_t cli_main; + + /* Node registrations added by constructors */ + vlib_node_registration_t *node_registrations; + + /* Event logger. */ + elog_main_t elog_main; + u32 configured_elog_ring_size; + + /* Packet trace capture filter */ + vlib_trace_filter_t trace_filter; + + /* List of init functions to call, setup by constructors */ + _vlib_init_function_list_elt_t *init_function_registrations; + _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; + _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; + _vlib_init_function_list_elt_t *worker_init_function_registrations; + _vlib_init_function_list_elt_t *api_init_function_registrations; + vlib_config_function_runtime_t *config_function_registrations; + + /* Hash table to record which init functions have been called. */ + uword *init_functions_called; + +} vlib_global_main_t; + /* Global main structure. */ -extern vlib_main_t vlib_global_main; +extern vlib_global_main_t vlib_global_main; void vlib_worker_loop (vlib_main_t * vm); always_inline f64 vlib_time_now (vlib_main_t * vm) { +#if CLIB_DEBUG > 0 + extern __thread uword __os_thread_index; +#endif + /* + * Make sure folks don't pass &vlib_global_main from a worker thread. + */ + ASSERT (vm->thread_index == __os_thread_index); return clib_time_now (&vm->clib_time) + vm->time_offset; } @@ -293,81 +389,85 @@ vlib_panic (vlib_main_t * vm) vlib_panic_with_error (vm, 0); } -always_inline u32 -vlib_vector_input_stats_index (vlib_main_t * vm, word delta) -{ - u32 i; - i = vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE; - ASSERT (is_pow2 (ARRAY_LEN (vm->vector_counts_per_main_loop))); - return (i + delta) & (ARRAY_LEN (vm->vector_counts_per_main_loop) - 1); -} - -/* Estimate input rate based on previous - 2^VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE - samples. */ -always_inline u32 -vlib_last_vectors_per_main_loop (vlib_main_t * vm) -{ - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 n = vm->vector_counts_per_main_loop[i]; - return n >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE; -} -/* Total ave vector count per iteration of main loop. */ always_inline f64 -vlib_last_vectors_per_main_loop_as_f64 (vlib_main_t * vm) +vlib_internal_node_vector_rate (vlib_main_t * vm) { - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 v = vm->vector_counts_per_main_loop[i]; - return (f64) v / (f64) (1 << VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE); + u64 vectors; + u64 calls; + + calls = vm->internal_node_calls - vm->internal_node_calls_last_clear; + + if (PREDICT_FALSE (calls == 0)) + return 0.0; + + vectors = vm->internal_node_vectors - vm->internal_node_vectors_last_clear; + + return (f64) vectors / (f64) calls; } -/* Total ave vectors/node count per iteration of main loop. */ -always_inline f64 -vlib_last_vector_length_per_node (vlib_main_t * vm) +always_inline void +vlib_clear_internal_node_vector_rate (vlib_main_t * vm) { - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 v = vm->vector_counts_per_main_loop[i]; - u32 n = vm->node_counts_per_main_loop[i]; - return n == 0 ? 0 : (f64) v / (f64) n; + vm->internal_node_calls_last_clear = vm->internal_node_calls; + vm->internal_node_vectors_last_clear = vm->internal_node_vectors; } -extern u32 wraps; - always_inline void vlib_increment_main_loop_counter (vlib_main_t * vm) { - u32 i, c, n, v, is_wrap; - - c = vm->main_loop_count++; - - is_wrap = (c & pow2_mask (VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)) == 0; - - if (is_wrap) - wraps++; - - i = vlib_vector_input_stats_index (vm, /* delta */ is_wrap); - - v = is_wrap ? 0 : vm->vector_counts_per_main_loop[i]; - n = is_wrap ? 0 : vm->node_counts_per_main_loop[i]; - - v += vm->main_loop_vectors_processed; - n += vm->main_loop_nodes_processed; - vm->main_loop_vectors_processed = 0; - vm->main_loop_nodes_processed = 0; - vm->vector_counts_per_main_loop[i] = v; - vm->node_counts_per_main_loop[i] = n; + vm->main_loop_count++; + vm->internal_node_last_vectors_per_main_loop = 0; if (PREDICT_FALSE (vm->main_loop_exit_now)) clib_longjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_CLI); } +always_inline u32 +vlib_last_vectors_per_main_loop (vlib_main_t * vm) +{ + return vm->internal_node_last_vectors_per_main_loop; +} + +always_inline void +vlib_node_runtime_perf_counter (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, uword n, u64 t, + vlib_node_runtime_perf_call_type_t call_type) +{ + vlib_node_runtime_perf_callback_data_t *v = + clib_callback_data_check_and_get (&vm->vlib_node_runtime_perf_callbacks); + if (vec_len (v)) + { + vlib_node_runtime_perf_callback_args_t args = { + .vm = vm, + .node = node, + .frame = frame, + .packets = n, + .cpu_time_now = t, + .call_type = call_type, + }; + clib_callback_data_call_vec (v, &args); + } +} + always_inline void vlib_set_queue_signal_callback (vlib_main_t * vm, void (*fp) (vlib_main_t *)) { vm->queue_signal_callback = fp; } +always_inline void +vlib_main_init () +{ + vlib_global_main_t *vgm = &vlib_global_main; + vlib_main_t *vm; + + vgm->init_functions_called = hash_create (0, /* value bytes */ 0); + + vm = clib_mem_alloc_aligned (sizeof (*vm), CLIB_CACHE_LINE_BYTES); + vec_add1 (vgm->vlib_mains, vm); +} + /* Main routine. */ int vlib_main (vlib_main_t * vm, unformat_input_t * input); @@ -377,10 +477,10 @@ extern u8 **vlib_thread_stacks; /* Number of thread stacks that the application needs */ u32 vlib_app_num_thread_stacks_needed (void) __attribute__ ((weak)); -extern void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n); +void vlib_add_del_post_mortem_callback (void *cb, int is_add); -#define VLIB_PCAP_MAJOR_VERSION 1 -#define VLIB_PCAP_MINOR_VERSION 0 +vlib_main_t *vlib_get_main_not_inline (void); +elog_main_t *vlib_get_elog_main_not_inline (); #endif /* included_vlib_main_h */