#ifndef included_vlib_main_h
#define included_vlib_main_h
+#include <vppinfra/callback_data.h>
#include <vppinfra/elog.h>
#include <vppinfra/format.h>
#include <vppinfra/longjmp.h>
#include <vppinfra/pool.h>
#include <vppinfra/random_buffer.h>
#include <vppinfra/time.h>
-#include <vppinfra/pmc.h>
-#include <vppinfra/pcap.h>
#include <pthread.h>
typedef struct
{
- /* Trace RX pkts */
- u8 pcap_rx_enable;
- /* Trace TX pkts */
- u8 pcap_tx_enable;
- /* Trace drop pkts */
- u8 pcap_drop_enable;
- u8 pad1;
- u32 max_bytes_per_pkt;
- u32 pcap_sw_if_index;
- pcap_main_t pcap_main;
- u32 filter_classify_table_index;
-} vnet_pcap_t;
+ u8 trace_filter_enable;
+ u32 classify_table_index;
+} vlib_trace_filter_t;
+
+typedef enum
+{
+ VLIB_NODE_RUNTIME_PERF_BEFORE,
+ VLIB_NODE_RUNTIME_PERF_AFTER,
+ VLIB_NODE_RUNTIME_PERF_RESET,
+} vlib_node_runtime_perf_call_type_t;
typedef struct
{
- u8 trace_filter_enable;
- u32 trace_classify_table_index;
- u32 trace_filter_set_index;
-} vlib_trace_filter_t;
+ struct vlib_main_t *vm;
+ vlib_node_runtime_t *node;
+ vlib_frame_t *frame;
+ uword packets;
+ u64 cpu_time_now;
+ vlib_node_runtime_perf_call_type_t call_type;
+} vlib_node_runtime_perf_callback_args_t;
+
+struct vlib_node_runtime_perf_callback_data_t;
+
+typedef void (*vlib_node_runtime_perf_callback_fp_t)
+ (struct vlib_node_runtime_perf_callback_data_t * data,
+ vlib_node_runtime_perf_callback_args_t * args);
+
+typedef struct vlib_node_runtime_perf_callback_data_t
+{
+ vlib_node_runtime_perf_callback_fp_t fp;
+ union
+ {
+ void *v;
+ u64 u;
+ } u[3];
+} vlib_node_runtime_perf_callback_data_t;
+
+clib_callback_data_typedef (vlib_node_runtime_perf_callback_set_t,
+ vlib_node_runtime_perf_callback_data_t);
typedef struct vlib_main_t
{
u64 cpu_time_main_loop_start;
/* Incremented once for each main loop. */
- u32 main_loop_count;
+ volatile u32 main_loop_count;
/* Count of vectors processed this main loop. */
u32 main_loop_vectors_processed;
u32 internal_node_last_vectors_per_main_loop;
/* Main loop hw / sw performance counters */
- void (**vlib_node_runtime_perf_counter_cbs) (struct vlib_main_t *,
- u64 *, u64 *,
- vlib_node_runtime_t *,
- vlib_frame_t *, int);
- void (**vlib_node_runtime_perf_counter_cb_tmp) (struct vlib_main_t *,
- u64 *, u64 *,
- vlib_node_runtime_t *,
- vlib_frame_t *, int);
+ vlib_node_runtime_perf_callback_set_t vlib_node_runtime_perf_callbacks;
+
+ /* dispatch wrapper function */
+ vlib_node_function_t *dispatch_wrapper_fn;
+
/* Every so often we switch to the next counter. */
#define VLIB_LOG2_MAIN_LOOPS_PER_STATS_UPDATE 7
u32 main_loop_exit_set;
/* Set e.g. in the SIGTERM signal handler, checked in a safe place... */
volatile u32 main_loop_exit_now;
+ /* Exit status that will be returned by the process upon exit. */
+ volatile int main_loop_exit_status;
clib_longjmp_t main_loop_exit;
#define VLIB_MAIN_LOOP_EXIT_NONE 0
#define VLIB_MAIN_LOOP_EXIT_PANIC 1
/* Error marker to use when exiting main loop. */
clib_error_t *main_loop_error;
- /* Name for e.g. syslog. */
- char *name;
-
/* Start of the heap. */
void *heap_base;
/* Node graph main structure. */
vlib_node_main_t node_main;
- /* Command line interface. */
- vlib_cli_main_t cli_main;
-
/* Packet trace buffer. */
vlib_trace_main_t trace_main;
- /* Pcap dispatch trace main */
- pcap_main_t dispatch_pcap_main;
- uword dispatch_pcap_enable;
- u32 *dispatch_buffer_trace_nodes;
- u8 *pcap_buffer;
-
- /* pcap rx / tx tracing */
- vnet_pcap_t pcap;
-
- /* Packet trace capture filter */
- vlib_trace_filter_t trace_filter;
-
/* Error handling. */
vlib_error_main_t error_main;
/* Stream index to use for distribution when MC is enabled. */
u32 mc_stream_index;
- vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join;
+ /* Hash table to record which init functions have been called. */
+ uword *worker_init_functions_called;
- /* Event logger. */
- elog_main_t elog_main;
+ vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join;
/* Event logger trace flags */
int elog_trace_api_messages;
/* Buffer of random data for various uses. */
clib_random_buffer_t random_buffer;
- /* Hash table to record which init functions have been called. */
- uword *init_functions_called;
-
/* thread, cpu and numa_node indices */
u32 thread_index;
u32 cpu_id;
u32 numa_node;
- /* List of init functions to call, setup by constructors */
- _vlib_init_function_list_elt_t *init_function_registrations;
- _vlib_init_function_list_elt_t *worker_init_function_registrations;
- _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
- _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
- _vlib_init_function_list_elt_t *api_init_function_registrations;
- vlib_config_function_runtime_t *config_function_registrations;
-
/* control-plane API queue signal pending, length indication */
volatile u32 queue_signal_pending;
volatile u32 api_queue_nonempty;
u8 **argv;
/* Top of (worker) dispatch loop callback */
- void (**volatile worker_thread_main_loop_callbacks) (struct vlib_main_t *);
+ void (**volatile worker_thread_main_loop_callbacks)
+ (struct vlib_main_t *, u64 t);
void (**volatile worker_thread_main_loop_callback_tmp)
- (struct vlib_main_t *);
+ (struct vlib_main_t *, u64 t);
clib_spinlock_t worker_thread_main_loop_callback_lock;
/* debugging */
volatile int parked_at_barrier;
- /* Attempt to do a post-mortem elog dump */
- int elog_post_mortem_dump;
-
- /*
- * Need to call vlib_worker_thread_node_runtime_update before
- * releasing worker thread barrier. Only valid in vlib_global_main.
- */
- int need_vlib_worker_thread_node_runtime_update;
-
/* Dispatch loop time accounting */
u64 loops_this_reporting_interval;
f64 loop_interval_end;
/* Earliest barrier can be closed again */
f64 barrier_no_close_before;
+ /* Barrier counter callback */
+ void (**volatile barrier_perf_callbacks)
+ (struct vlib_main_t *, u64 t, int leave);
+ void (**volatile barrier_perf_callbacks_tmp)
+ (struct vlib_main_t *, u64 t, int leave);
+
/* Need to check the frame queues */
volatile uword check_frame_queues;
u32 buffer_alloc_success_seed;
f64 buffer_alloc_success_rate;
+#ifdef CLIB_SANITIZE_ADDR
+ /* address sanitizer stack save */
+ void *asan_stack_save;
+#endif
} vlib_main_t;
+typedef struct vlib_global_main_t
+{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+
+ /* Per-thread Mains */
+ vlib_main_t **vlib_mains;
+
+ /* Name for e.g. syslog. */
+ char *name;
+
+ /* post-mortem callbacks */
+ void (**post_mortem_callbacks) (void);
+
+ /*
+ * Need to call vlib_worker_thread_node_runtime_update before
+ * releasing worker thread barrier.
+ */
+ int need_vlib_worker_thread_node_runtime_update;
+
+ /* Command line interface. */
+ vlib_cli_main_t cli_main;
+
+ /* Node registrations added by constructors */
+ vlib_node_registration_t *node_registrations;
+
+ /* Event logger. */
+ elog_main_t elog_main;
+ u32 configured_elog_ring_size;
+
+ /* Packet trace capture filter */
+ vlib_trace_filter_t trace_filter;
+
+ /* List of init functions to call, setup by constructors */
+ _vlib_init_function_list_elt_t *init_function_registrations;
+ _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
+ _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
+ _vlib_init_function_list_elt_t *worker_init_function_registrations;
+ _vlib_init_function_list_elt_t *api_init_function_registrations;
+ vlib_config_function_runtime_t *config_function_registrations;
+
+ /* Hash table to record which init functions have been called. */
+ uword *init_functions_called;
+
+} vlib_global_main_t;
+
/* Global main structure. */
-extern vlib_main_t vlib_global_main;
+extern vlib_global_main_t vlib_global_main;
void vlib_worker_loop (vlib_main_t * vm);
vlib_panic_with_error (vm, 0);
}
+/* Asynchronously requests exit with the given status. */
+void vlib_exit_with_status (vlib_main_t *vm, int status);
always_inline f64
vlib_internal_node_vector_rate (vlib_main_t * vm)
return vm->internal_node_last_vectors_per_main_loop;
}
+always_inline void
+vlib_node_runtime_perf_counter (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame, uword n, u64 t,
+ vlib_node_runtime_perf_call_type_t call_type)
+{
+ vlib_node_runtime_perf_callback_data_t *v =
+ clib_callback_data_check_and_get (&vm->vlib_node_runtime_perf_callbacks);
+ if (vec_len (v))
+ {
+ vlib_node_runtime_perf_callback_args_t args = {
+ .vm = vm,
+ .node = node,
+ .frame = frame,
+ .packets = n,
+ .cpu_time_now = t,
+ .call_type = call_type,
+ };
+ clib_callback_data_call_vec (v, &args);
+ }
+}
+
always_inline void vlib_set_queue_signal_callback
(vlib_main_t * vm, void (*fp) (vlib_main_t *))
{
vm->queue_signal_callback = fp;
}
+always_inline void
+vlib_main_init ()
+{
+ vlib_global_main_t *vgm = &vlib_global_main;
+ vlib_main_t *vm;
+
+ vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
+
+ vm = clib_mem_alloc_aligned (sizeof (*vm), CLIB_CACHE_LINE_BYTES);
+ vec_add1 (vgm->vlib_mains, vm);
+}
+
/* Main routine. */
int vlib_main (vlib_main_t * vm, unformat_input_t * input);
/* Number of thread stacks that the application needs */
u32 vlib_app_num_thread_stacks_needed (void) __attribute__ ((weak));
-extern void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n);
+void vlib_add_del_post_mortem_callback (void *cb, int is_add);
-#define VLIB_PCAP_MAJOR_VERSION 1
-#define VLIB_PCAP_MINOR_VERSION 0
-
-typedef struct
-{
- u8 *filename;
- int enable;
- int status;
- u32 packets_to_capture;
- u32 buffer_trace_node_index;
- u32 buffer_traces_to_capture;
-} vlib_pcap_dispatch_trace_args_t;
-
-int vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t *);
+vlib_main_t *vlib_get_main_not_inline (void);
+elog_main_t *vlib_get_elog_main_not_inline ();
#endif /* included_vlib_main_h */