X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fmain.h;h=5f78b7941a4f2bd9b6cbe4bbb30e1cb29bbe6c5d;hb=8b60fb0fe;hp=bfa7ddbe6afd95911f3e58000c88452723499f7e;hpb=814813103bb4acb9ced39c22972bd5e97df13d33;p=vpp.git diff --git a/src/vlib/main.h b/src/vlib/main.h index bfa7ddbe6af..5f78b7941a4 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -40,12 +40,14 @@ #ifndef included_vlib_main_h #define included_vlib_main_h +#include #include #include #include #include #include #include +#include #include @@ -56,10 +58,71 @@ #define VLIB_ELOG_MAIN_LOOP 0 #endif +typedef struct +{ + /* Trace RX pkts */ + u8 pcap_rx_enable; + /* Trace TX pkts */ + u8 pcap_tx_enable; + /* Trace drop pkts */ + u8 pcap_drop_enable; + u8 pad1; + u32 max_bytes_per_pkt; + u32 pcap_sw_if_index; + pcap_main_t pcap_main; + u32 filter_classify_table_index; +} vnet_pcap_t; + +typedef struct +{ + u8 trace_filter_enable; + u32 classify_table_index; +} vlib_trace_filter_t; + +typedef enum +{ + VLIB_NODE_RUNTIME_PERF_BEFORE, + VLIB_NODE_RUNTIME_PERF_AFTER, + VLIB_NODE_RUNTIME_PERF_RESET, +} vlib_node_runtime_perf_call_type_t; + +typedef struct +{ + struct vlib_main_t *vm; + vlib_node_runtime_t *node; + vlib_frame_t *frame; + uword packets; + u64 cpu_time_now; + vlib_node_runtime_perf_call_type_t call_type; +} vlib_node_runtime_perf_callback_args_t; + +struct vlib_node_runtime_perf_callback_data_t; + +typedef void (*vlib_node_runtime_perf_callback_fp_t) + (struct vlib_node_runtime_perf_callback_data_t * data, + vlib_node_runtime_perf_callback_args_t * args); + +typedef struct vlib_node_runtime_perf_callback_data_t +{ + vlib_node_runtime_perf_callback_fp_t fp; + union + { + void *v; + u64 u; + } u[3]; +} vlib_node_runtime_perf_callback_data_t; + +clib_callback_data_typedef (vlib_node_runtime_perf_callback_set_t, + vlib_node_runtime_perf_callback_data_t); + typedef struct vlib_main_t { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* Instruction level timing state. */ clib_time_t clib_time; + /* Offset from main thread time */ + f64 time_offset; + f64 time_last_barrier_release; /* Time stamp of last node dispatch. */ u64 cpu_time_last_node_dispatch; @@ -68,23 +131,34 @@ typedef struct vlib_main_t u64 cpu_time_main_loop_start; /* Incremented once for each main loop. */ - u32 main_loop_count; + volatile u32 main_loop_count; /* Count of vectors processed this main loop. */ u32 main_loop_vectors_processed; u32 main_loop_nodes_processed; - /* Circular buffer of input node vector counts. - Indexed by low bits of - (main_loop_count >> VLIB_LOG2_INPUT_VECTORS_PER_MAIN_LOOP). */ - u32 vector_counts_per_main_loop[2]; - u32 node_counts_per_main_loop[2]; + /* Internal node vectors, calls */ + u64 internal_node_vectors; + u64 internal_node_calls; + u64 internal_node_vectors_last_clear; + u64 internal_node_calls_last_clear; + + /* Instantaneous vector rate */ + u32 internal_node_last_vectors_per_main_loop; + + /* Main loop hw / sw performance counters */ + vlib_node_runtime_perf_callback_set_t vlib_node_runtime_perf_callbacks; + + /* dispatch wrapper function */ + vlib_node_function_t *dispatch_wrapper_fn; /* Every so often we switch to the next counter. */ #define VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE 7 /* Jump target to exit main loop with given code. */ u32 main_loop_exit_set; + /* Set e.g. in the SIGTERM signal handler, checked in a safe place... */ + volatile u32 main_loop_exit_now; clib_longjmp_t main_loop_exit; #define VLIB_MAIN_LOOP_EXIT_NONE 0 #define VLIB_MAIN_LOOP_EXIT_PANIC 1 @@ -97,20 +171,21 @@ typedef struct vlib_main_t /* Name for e.g. syslog. */ char *name; - /* Start and size of CLIB heap. */ + /* Start of the heap. */ void *heap_base; + + /* Truncated version, to create frame indices */ + void *heap_aligned_base; + + /* Size of the heap */ uword heap_size; + /* buffer main structure. */ vlib_buffer_main_t *buffer_main; + /* physical memory main structure. */ vlib_physmem_main_t physmem_main; - /* Allocate/free buffer memory for DMA transfers, descriptor rings, etc. - buffer memory is guaranteed to be cache-aligned. */ - void *(*os_physmem_alloc_aligned) (vlib_physmem_main_t * pm, - uword n_bytes, uword alignment); - void (*os_physmem_free) (void *x); - /* Node graph main structure. */ vlib_node_main_t node_main; @@ -120,6 +195,19 @@ typedef struct vlib_main_t /* Packet trace buffer. */ vlib_trace_main_t trace_main; + /* Pcap dispatch trace main */ + pcap_main_t dispatch_pcap_main; + uword dispatch_pcap_enable; + uword dispatch_pcap_postmortem; + u32 *dispatch_buffer_trace_nodes; + u8 *pcap_buffer; + + /* pcap rx / tx tracing */ + vnet_pcap_t pcap; + + /* Packet trace capture filter */ + vlib_trace_filter_t trace_filter; + /* Error handling. */ vlib_error_main_t error_main; @@ -129,9 +217,6 @@ typedef struct vlib_main_t struct vlib_node_runtime_t * node, vlib_frame_t * frame); - /* Multicast distribution. Set to zero for MC disabled. */ - mc_main_t *mc_main; - /* Stream index to use for distribution when MC is enabled. */ u32 mc_stream_index; @@ -139,6 +224,14 @@ typedef struct vlib_main_t /* Event logger. */ elog_main_t elog_main; + u32 configured_elog_ring_size; + + /* Event logger trace flags */ + int elog_trace_api_messages; + int elog_trace_cli_commands; + int elog_trace_graph_dispatch; + int elog_trace_graph_circuit; + u32 elog_trace_graph_circuit_node_index; /* Node call and return event types. */ elog_event_type_t *node_call_elog_event_types; @@ -155,10 +248,10 @@ typedef struct vlib_main_t /* Hash table to record which init functions have been called. */ uword *init_functions_called; - /* to compare with node runtime */ + /* thread, cpu and numa_node indices */ u32 thread_index; - - void **mbuf_alloc_list; + u32 cpu_id; + u32 numa_node; /* List of init functions to call, setup by constructors */ _vlib_init_function_list_elt_t *init_function_registrations; @@ -167,7 +260,6 @@ typedef struct vlib_main_t _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; _vlib_init_function_list_elt_t *api_init_function_registrations; vlib_config_function_runtime_t *config_function_registrations; - mc_serialize_msg_t *mc_msg_registrations; /* mc_main is a pointer... */ /* control-plane API queue signal pending, length indication */ volatile u32 queue_signal_pending; @@ -175,12 +267,64 @@ typedef struct vlib_main_t void (*queue_signal_callback) (struct vlib_main_t *); u8 **argv; + /* Top of (worker) dispatch loop callback */ + void (**volatile worker_thread_main_loop_callbacks) + (struct vlib_main_t *, u64 t); + void (**volatile worker_thread_main_loop_callback_tmp) + (struct vlib_main_t *, u64 t); + clib_spinlock_t worker_thread_main_loop_callback_lock; + /* debugging */ volatile int parked_at_barrier; /* Attempt to do a post-mortem elog dump */ int elog_post_mortem_dump; + /* + * Need to call vlib_worker_thread_node_runtime_update before + * releasing worker thread barrier. Only valid in vlib_global_main. + */ + int need_vlib_worker_thread_node_runtime_update; + + /* Dispatch loop time accounting */ + u64 loops_this_reporting_interval; + f64 loop_interval_end; + f64 loop_interval_start; + f64 loops_per_second; + f64 seconds_per_loop; + f64 damping_constant; + + /* + * Barrier epoch - Set to current time, each time barrier_sync or + * barrier_release is called with zero recursion. + */ + f64 barrier_epoch; + + /* Earliest barrier can be closed again */ + f64 barrier_no_close_before; + + /* Barrier counter callback */ + void (**volatile barrier_perf_callbacks) + (struct vlib_main_t *, u64 t, int leave); + void (**volatile barrier_perf_callbacks_tmp) + (struct vlib_main_t *, u64 t, int leave); + + /* Need to check the frame queues */ + volatile uword check_frame_queues; + + /* RPC requests, main thread only */ + uword *pending_rpc_requests; + uword *processing_rpc_requests; + clib_spinlock_t pending_rpc_lock; + + /* buffer fault injector */ + u32 buffer_alloc_success_seed; + f64 buffer_alloc_success_rate; + +#ifdef CLIB_SANITIZE_ADDR + /* address sanitizer stack save */ + void *asan_stack_save; +#endif } vlib_main_t; /* Global main structure. */ @@ -191,7 +335,14 @@ void vlib_worker_loop (vlib_main_t * vm); always_inline f64 vlib_time_now (vlib_main_t * vm) { - return clib_time_now (&vm->clib_time); +#if CLIB_DEBUG > 0 + extern __thread uword __os_thread_index; +#endif + /* + * Make sure folks don't pass &vlib_global_main from a worker thread. + */ + ASSERT (vm->thread_index == __os_thread_index); + return clib_time_now (&vm->clib_time) + vm->time_offset; } always_inline f64 @@ -249,70 +400,65 @@ vlib_panic (vlib_main_t * vm) vlib_panic_with_error (vm, 0); } -always_inline u32 -vlib_vector_input_stats_index (vlib_main_t * vm, word delta) -{ - u32 i; - i = vm->main_loop_count >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE; - ASSERT (is_pow2 (ARRAY_LEN (vm->vector_counts_per_main_loop))); - return (i + delta) & (ARRAY_LEN (vm->vector_counts_per_main_loop) - 1); -} - -/* Estimate input rate based on previous - 2^VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE - samples. */ -always_inline u32 -vlib_last_vectors_per_main_loop (vlib_main_t * vm) -{ - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 n = vm->vector_counts_per_main_loop[i]; - return n >> VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE; -} -/* Total ave vector count per iteration of main loop. */ always_inline f64 -vlib_last_vectors_per_main_loop_as_f64 (vlib_main_t * vm) +vlib_internal_node_vector_rate (vlib_main_t * vm) { - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 v = vm->vector_counts_per_main_loop[i]; - return (f64) v / (f64) (1 << VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE); + u64 vectors; + u64 calls; + + calls = vm->internal_node_calls - vm->internal_node_calls_last_clear; + + if (PREDICT_FALSE (calls == 0)) + return 0.0; + + vectors = vm->internal_node_vectors - vm->internal_node_vectors_last_clear; + + return (f64) vectors / (f64) calls; } -/* Total ave vectors/node count per iteration of main loop. */ -always_inline f64 -vlib_last_vector_length_per_node (vlib_main_t * vm) +always_inline void +vlib_clear_internal_node_vector_rate (vlib_main_t * vm) { - u32 i = vlib_vector_input_stats_index (vm, -1); - u32 v = vm->vector_counts_per_main_loop[i]; - u32 n = vm->node_counts_per_main_loop[i]; - return n == 0 ? 0 : (f64) v / (f64) n; + vm->internal_node_calls_last_clear = vm->internal_node_calls; + vm->internal_node_vectors_last_clear = vm->internal_node_vectors; } -extern u32 wraps; - always_inline void vlib_increment_main_loop_counter (vlib_main_t * vm) { - u32 i, c, n, v, is_wrap; + vm->main_loop_count++; + vm->internal_node_last_vectors_per_main_loop = 0; - c = vm->main_loop_count++; - - is_wrap = (c & pow2_mask (VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE)) == 0; - - if (is_wrap) - wraps++; - - i = vlib_vector_input_stats_index (vm, /* delta */ is_wrap); + if (PREDICT_FALSE (vm->main_loop_exit_now)) + clib_longjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_CLI); +} - v = is_wrap ? 0 : vm->vector_counts_per_main_loop[i]; - n = is_wrap ? 0 : vm->node_counts_per_main_loop[i]; +always_inline u32 +vlib_last_vectors_per_main_loop (vlib_main_t * vm) +{ + return vm->internal_node_last_vectors_per_main_loop; +} - v += vm->main_loop_vectors_processed; - n += vm->main_loop_nodes_processed; - vm->main_loop_vectors_processed = 0; - vm->main_loop_nodes_processed = 0; - vm->vector_counts_per_main_loop[i] = v; - vm->node_counts_per_main_loop[i] = n; +always_inline void +vlib_node_runtime_perf_counter (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, uword n, u64 t, + vlib_node_runtime_perf_call_type_t call_type) +{ + vlib_node_runtime_perf_callback_data_t *v = + clib_callback_data_check_and_get (&vm->vlib_node_runtime_perf_callbacks); + if (vec_len (v)) + { + vlib_node_runtime_perf_callback_args_t args = { + .vm = vm, + .node = node, + .frame = frame, + .packets = n, + .cpu_time_now = t, + .call_type = call_type, + }; + clib_callback_data_call_vec (v, &args); + } } always_inline void vlib_set_queue_signal_callback @@ -332,6 +478,24 @@ u32 vlib_app_num_thread_stacks_needed (void) __attribute__ ((weak)); extern void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n); +#define VLIB_PCAP_MAJOR_VERSION 1 +#define VLIB_PCAP_MINOR_VERSION 0 + +typedef struct +{ + u8 *filename; + int enable; + int status; + int post_mortem; + u32 packets_to_capture; + u32 buffer_trace_node_index; + u32 buffer_traces_to_capture; +} vlib_pcap_dispatch_trace_args_t; + +int vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t *); +vlib_main_t *vlib_get_main_not_inline (void); +elog_main_t *vlib_get_elog_main_not_inline (); + #endif /* included_vlib_main_h */ /*