X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fmain.c;h=bee63970f3d15c0b26230efc6dee9fc35781217e;hb=4ac258497303c1cbca539e04ef5f732eed24a5c4;hp=8d7c6c0927528b5663bdd010e8da109557636930;hpb=d25147d58117a573fd2fc04cdfa76339fa1cdf22;p=vpp.git diff --git a/src/vlib/main.c b/src/vlib/main.c index 8d7c6c09275..bee63970f3d 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -122,6 +122,8 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t *f; u32 l, n, scalar_size, vector_size; + ASSERT (vm == vlib_get_main ()); + to_node = vlib_get_node (vm, to_node_index); scalar_size = to_node->scalar_size; @@ -221,6 +223,8 @@ vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f) if (f->n_vectors == 0) return; + ASSERT (vm == vlib_get_main ()); + vlib_validate_frame_indices (f); to_node = vlib_get_node (vm, to_node_index); @@ -241,6 +245,7 @@ vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f) vlib_node_t *node; vlib_frame_size_t *fs; + ASSERT (vm == vlib_get_main ()); ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); node = vlib_get_node (vm, r->node_index); @@ -565,44 +570,38 @@ vlib_put_next_frame (vlib_main_t * vm, } /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */ -never_inline void -vlib_node_runtime_sync_stats (vlib_main_t * vm, - vlib_node_runtime_t * r, - uword n_calls, uword n_vectors, uword n_clocks, - uword n_ticks0, uword n_ticks1) +void +vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r, + uword n_calls, uword n_vectors, + uword n_clocks) { - vlib_node_t *n = vlib_get_node (vm, r->node_index); - n->stats_total.calls += n_calls + r->calls_since_last_overflow; n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow; n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow; - n->stats_total.perf_counter0_ticks += n_ticks0 + - r->perf_counter0_ticks_since_last_overflow; - n->stats_total.perf_counter1_ticks += n_ticks1 + - r->perf_counter1_ticks_since_last_overflow; - n->stats_total.perf_counter_vectors += n_vectors + - r->perf_counter_vectors_since_last_overflow; n->stats_total.max_clock = r->max_clock; n->stats_total.max_clock_n = r->max_clock_n; r->calls_since_last_overflow = 0; r->vectors_since_last_overflow = 0; r->clocks_since_last_overflow = 0; - r->perf_counter0_ticks_since_last_overflow = 0ULL; - r->perf_counter1_ticks_since_last_overflow = 0ULL; - r->perf_counter_vectors_since_last_overflow = 0ULL; +} + +void +vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r, + uword n_calls, uword n_vectors, uword n_clocks) +{ + vlib_node_t *n = vlib_get_node (vm, r->node_index); + vlib_node_runtime_sync_stats_node (n, r, n_calls, n_vectors, n_clocks); } always_inline void __attribute__ ((unused)) vlib_process_sync_stats (vlib_main_t * vm, vlib_process_t * p, - uword n_calls, uword n_vectors, uword n_clocks, - uword n_ticks0, uword n_ticks1) + uword n_calls, uword n_vectors, uword n_clocks) { vlib_node_runtime_t *rt = &p->node_runtime; vlib_node_t *n = vlib_get_node (vm, rt->node_index); - vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks, - n_ticks0, n_ticks1); + vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks); n->stats_total.suspends += p->n_suspends; p->n_suspends = 0; } @@ -615,7 +614,7 @@ vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) if (n->type == VLIB_NODE_TYPE_PROCESS) { /* Nothing to do for PROCESS nodes except in main thread */ - if (vm != &vlib_global_main) + if (vm != vlib_get_first_main ()) return; vlib_process_t *p = vlib_get_process_from_node (vm, n); @@ -628,7 +627,7 @@ vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) vec_elt_at_index (vm->node_main.nodes_by_type[n->type], n->runtime_index); - vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0, 0); + vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0); /* Sync up runtime next frame vector counters with main node structure. */ { @@ -648,32 +647,21 @@ always_inline u32 vlib_node_runtime_update_stats (vlib_main_t * vm, vlib_node_runtime_t * node, uword n_calls, - uword n_vectors, uword n_clocks, - uword n_ticks0, uword n_ticks1) + uword n_vectors, uword n_clocks) { u32 ca0, ca1, v0, v1, cl0, cl1, r; - u32 ptick00, ptick01, ptick10, ptick11, pvec0, pvec1; cl0 = cl1 = node->clocks_since_last_overflow; ca0 = ca1 = node->calls_since_last_overflow; v0 = v1 = node->vectors_since_last_overflow; - ptick00 = ptick01 = node->perf_counter0_ticks_since_last_overflow; - ptick10 = ptick11 = node->perf_counter1_ticks_since_last_overflow; - pvec0 = pvec1 = node->perf_counter_vectors_since_last_overflow; ca1 = ca0 + n_calls; v1 = v0 + n_vectors; cl1 = cl0 + n_clocks; - ptick01 = ptick00 + n_ticks0; - ptick11 = ptick10 + n_ticks1; - pvec1 = pvec0 + n_vectors; node->calls_since_last_overflow = ca1; node->clocks_since_last_overflow = cl1; node->vectors_since_last_overflow = v1; - node->perf_counter0_ticks_since_last_overflow = ptick01; - node->perf_counter1_ticks_since_last_overflow = ptick11; - node->perf_counter_vectors_since_last_overflow = pvec1; node->max_clock_n = node->max_clock > n_clocks ? node->max_clock_n : n_vectors; @@ -681,49 +669,32 @@ vlib_node_runtime_update_stats (vlib_main_t * vm, r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors); - if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick01 < ptick00) - || (ptick11 < ptick10) || (pvec1 < pvec0)) + if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0)) { node->calls_since_last_overflow = ca0; node->clocks_since_last_overflow = cl0; node->vectors_since_last_overflow = v0; - node->perf_counter0_ticks_since_last_overflow = ptick00; - node->perf_counter1_ticks_since_last_overflow = ptick10; - node->perf_counter_vectors_since_last_overflow = pvec0; - vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks, - n_ticks0, n_ticks1); + vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks); } return r; } -always_inline void -vlib_node_runtime_perf_counter (vlib_main_t * vm, u64 * pmc0, u64 * pmc1, - vlib_node_runtime_t * node, - vlib_frame_t * frame, int before_or_after) -{ - *pmc0 = 0; - *pmc1 = 0; - if (PREDICT_FALSE (vec_len (vm->vlib_node_runtime_perf_counter_cbs) != 0)) - clib_call_callbacks (vm->vlib_node_runtime_perf_counter_cbs, vm, pmc0, - pmc1, node, frame, before_or_after); -} - always_inline void vlib_process_update_stats (vlib_main_t * vm, vlib_process_t * p, uword n_calls, uword n_vectors, uword n_clocks) { vlib_node_runtime_update_stats (vm, &p->node_runtime, - n_calls, n_vectors, n_clocks, 0ULL, 0ULL); + n_calls, n_vectors, n_clocks); } static clib_error_t * vlib_cli_elog_clear (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - elog_reset_buffer (&vm->elog_main); + elog_reset_buffer (&vlib_global_main.elog_main); return 0; } @@ -740,7 +711,7 @@ static clib_error_t * elog_save_buffer (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - elog_main_t *em = &vm->elog_main; + elog_main_t *em = &vlib_global_main.elog_main; char *file, *chroot_file; clib_error_t *error = 0; @@ -774,21 +745,12 @@ elog_save_buffer (vlib_main_t * vm, } void -elog_post_mortem_dump (void) +vlib_post_mortem_dump (void) { - vlib_main_t *vm = &vlib_global_main; - elog_main_t *em = &vm->elog_main; - u8 *filename; - clib_error_t *error; - - if (!vm->elog_post_mortem_dump) - return; + vlib_global_main_t *vgm = vlib_get_global_main (); - filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0); - error = elog_write_file (em, (char *) filename, 1 /* flush ring */ ); - if (error) - clib_error_report (error); - vec_free (filename); + for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) + (vgm->post_mortem_callbacks[i]) (); } /* *INDENT-OFF* */ @@ -803,7 +765,7 @@ static clib_error_t * elog_stop (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - elog_main_t *em = &vm->elog_main; + elog_main_t *em = &vlib_global_main.elog_main; em->n_total_events_disable_limit = em->n_total_events; @@ -823,7 +785,7 @@ static clib_error_t * elog_restart (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - elog_main_t *em = &vm->elog_main; + elog_main_t *em = &vlib_global_main.elog_main; em->n_total_events_disable_limit = ~0; @@ -840,14 +802,14 @@ VLIB_CLI_COMMAND (elog_restart_cli, static) = { /* *INDENT-ON* */ static clib_error_t * -elog_resize (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) +elog_resize_command_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) { - elog_main_t *em = &vm->elog_main; + elog_main_t *em = &vlib_global_main.elog_main; u32 tmp; /* Stop the parade */ - elog_reset_buffer (&vm->elog_main); + elog_reset_buffer (em); if (unformat (input, "%d", &tmp)) { @@ -865,7 +827,7 @@ elog_resize (vlib_main_t * vm, VLIB_CLI_COMMAND (elog_resize_cli, static) = { .path = "event-logger resize", .short_help = "event-logger resize ", - .function = elog_resize, + .function = elog_resize_command_fn, }; /* *INDENT-ON* */ @@ -874,7 +836,7 @@ VLIB_CLI_COMMAND (elog_resize_cli, static) = { static void elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show) { - elog_main_t *em = &vm->elog_main; + elog_main_t *em = &vlib_global_main.elog_main; elog_event_t *e, *es; f64 dt; @@ -939,8 +901,8 @@ vlib_elog_main_loop_event (vlib_main_t * vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return) { - vlib_main_t *evm = &vlib_global_main; - elog_main_t *em = &evm->elog_main; + vlib_main_t *evm = vlib_get_first_main (); + elog_main_t *em = vlib_get_elog_main (); int enabled = evm->elog_trace_graph_dispatch | evm->elog_trace_graph_circuit; @@ -998,163 +960,6 @@ add_trajectory_trace (vlib_buffer_t * b, u32 node_index) #endif } -u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak)); -u8 * -format_vnet_buffer_flags (u8 * s, va_list * args) -{ - s = format (s, "BUG STUB %s", __FUNCTION__); - return s; -} - -u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak)); -u8 * -format_vnet_buffer_opaque (u8 * s, va_list * args) -{ - s = format (s, "BUG STUB %s", __FUNCTION__); - return s; -} - -u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args) - __attribute__ ((weak)); -u8 * -format_vnet_buffer_opaque2 (u8 * s, va_list * args) -{ - s = format (s, "BUG STUB %s", __FUNCTION__); - return s; -} - -static u8 * -format_buffer_metadata (u8 * s, va_list * args) -{ - vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); - - s = format (s, "flags: %U\n", format_vnet_buffer_flags, b); - s = format (s, "current_data: %d, current_length: %d\n", - (i32) (b->current_data), (i32) (b->current_length)); - s = format - (s, - "current_config_index/punt_reason: %d, flow_id: %x, next_buffer: %x\n", - b->current_config_index, b->flow_id, b->next_buffer); - s = - format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n", - (u32) (b->error), (u32) (b->ref_count), - (u32) (b->buffer_pool_index)); - s = - format (s, "trace_handle: 0x%x, len_not_first_buf: %d\n", b->trace_handle, - b->total_length_not_including_first_buffer); - return s; -} - -#define A(x) vec_add1(vm->pcap_buffer, (x)) - -static void -dispatch_pcap_trace (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - int i; - vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b; - pcap_main_t *pm = &vlib_global_main.dispatch_pcap_main; - vlib_trace_main_t *tm = &vm->trace_main; - u32 capture_size; - vlib_node_t *n; - i32 n_left; - f64 time_now = vlib_time_now (vm); - u32 *from; - u8 *d; - u8 string_count; - - /* Input nodes don't have frames yet */ - if (frame == 0 || frame->n_vectors == 0) - return; - - from = vlib_frame_vector_args (frame); - vlib_get_buffers (vm, from, bufs, frame->n_vectors); - bufp = bufs; - - n = vlib_get_node (vm, node->node_index); - - for (i = 0; i < frame->n_vectors; i++) - { - if (PREDICT_TRUE (pm->n_packets_captured < pm->n_packets_to_capture)) - { - b = bufp[i]; - - vec_reset_length (vm->pcap_buffer); - string_count = 0; - - /* Version, flags */ - A ((u8) VLIB_PCAP_MAJOR_VERSION); - A ((u8) VLIB_PCAP_MINOR_VERSION); - A (0 /* string_count */ ); - A (n->protocol_hint); - - /* Buffer index (big endian) */ - A ((from[i] >> 24) & 0xff); - A ((from[i] >> 16) & 0xff); - A ((from[i] >> 8) & 0xff); - A ((from[i] >> 0) & 0xff); - - /* Node name, NULL-terminated ASCII */ - vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0); - string_count++; - - vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", - format_buffer_metadata, b, 0); - string_count++; - vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", - format_vnet_buffer_opaque, b, 0); - string_count++; - vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", - format_vnet_buffer_opaque2, b, 0); - string_count++; - - /* Is this packet traced? */ - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED)) - { - vlib_trace_header_t **h - = pool_elt_at_index (tm->trace_buffer_pool, - vlib_buffer_get_trace_index (b)); - - vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", - format_vlib_trace, vm, h[0], 0); - string_count++; - } - - /* Save the string count */ - vm->pcap_buffer[2] = string_count; - - /* Figure out how many bytes in the pcap trace */ - capture_size = vec_len (vm->pcap_buffer) + - +vlib_buffer_length_in_chain (vm, b); - - clib_spinlock_lock_if_init (&pm->lock); - n_left = clib_min (capture_size, 16384); - d = pcap_add_packet (pm, time_now, n_left, capture_size); - - /* Copy the header */ - clib_memcpy_fast (d, vm->pcap_buffer, vec_len (vm->pcap_buffer)); - d += vec_len (vm->pcap_buffer); - - n_left = clib_min - (vlib_buffer_length_in_chain (vm, b), - (16384 - vec_len (vm->pcap_buffer))); - /* Copy the packet data */ - while (1) - { - u32 copy_length = clib_min ((u32) n_left, b->current_length); - clib_memcpy_fast (d, b->data + b->current_data, copy_length); - n_left -= b->current_length; - if (n_left <= 0) - break; - d += b->current_length; - ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT); - b = vlib_get_buffer (vm, b->next_buffer); - } - clib_spinlock_unlock_if_init (&pm->lock); - } - } -} - static_always_inline u64 dispatch_node (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1166,7 +971,6 @@ dispatch_node (vlib_main_t * vm, u64 t; vlib_node_main_t *nm = &vm->node_main; vlib_next_frame_t *nf; - u64 pmc_before[2], pmc_after[2], pmc_delta[2]; if (CLIB_DEBUG > 0) { @@ -1206,8 +1010,8 @@ dispatch_node (vlib_main_t * vm, last_time_stamp, frame ? frame->n_vectors : 0, /* is_after */ 0); - vlib_node_runtime_perf_counter (vm, &pmc_before[0], &pmc_before[1], - node, frame, 0 /* before */ ); + vlib_node_runtime_perf_counter (vm, node, frame, 0, last_time_stamp, + VLIB_NODE_RUNTIME_PERF_BEFORE); /* * Turn this on if you run into @@ -1224,28 +1028,23 @@ dispatch_node (vlib_main_t * vm, vlib_buffer_t *b = vlib_get_buffer (vm, from[i]); add_trajectory_trace (b, node->node_index); } - if (PREDICT_FALSE (vm->dispatch_pcap_enable)) - dispatch_pcap_trace (vm, node, frame); - n = node->function (vm, node, frame); + if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) + n = node->function (vm, node, frame); + else + n = vm->dispatch_wrapper_fn (vm, node, frame); } else { - if (PREDICT_FALSE (vm->dispatch_pcap_enable)) - dispatch_pcap_trace (vm, node, frame); - n = node->function (vm, node, frame); + if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) + n = node->function (vm, node, frame); + else + n = vm->dispatch_wrapper_fn (vm, node, frame); } t = clib_cpu_time_now (); - /* - * To validate accounting: pmc_delta = t - pmc_before; - * perf ticks should equal clocks/pkt... - */ - vlib_node_runtime_perf_counter (vm, &pmc_after[0], &pmc_after[1], node, - frame, 1 /* after */ ); - - pmc_delta[0] = pmc_after[0] - pmc_before[0]; - pmc_delta[1] = pmc_after[1] - pmc_before[1]; + vlib_node_runtime_perf_counter (vm, node, frame, n, t, + VLIB_NODE_RUNTIME_PERF_AFTER); vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ ); @@ -1255,17 +1054,11 @@ dispatch_node (vlib_main_t * vm, v = vlib_node_runtime_update_stats (vm, node, /* n_calls */ 1, /* n_vectors */ n, - /* n_clocks */ t - last_time_stamp, - pmc_delta[0] /* PMC0 */ , - pmc_delta[1] /* PMC1 */ ); - - /* When in interrupt mode and vector rate crosses threshold switch to - polling mode. */ - if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT) - || (dispatch_state == VLIB_NODE_STATE_POLLING - && (node->flags - & - VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)))) + /* n_clocks */ t - last_time_stamp); + + /* When in adaptive mode and vector rate crosses threshold switch to + polling mode and vice versa. */ + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE)) { /* *INDENT-OFF* */ ELOG_TYPE_DECLARE (e) = @@ -1298,7 +1091,8 @@ dispatch_node (vlib_main_t * vm, nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1; nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1; - if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) + if (PREDICT_FALSE ( + vlib_get_first_main ()->elog_trace_graph_dispatch)) { vlib_worker_thread_t *w = vlib_worker_threads + vm->thread_index; @@ -1333,7 +1127,8 @@ dispatch_node (vlib_main_t * vm, + vm->thread_index; node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; - if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) + if (PREDICT_FALSE ( + vlib_get_first_main ()->elog_trace_graph_dispatch)) { ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track); @@ -1354,7 +1149,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, { vlib_node_main_t *nm = &vm->node_main; vlib_frame_t *f; - vlib_next_frame_t *nf, nf_dummy; + vlib_next_frame_t *nf, nf_placeholder; vlib_node_runtime_t *n; vlib_frame_t *restore_frame; vlib_pending_frame_t *p; @@ -1368,8 +1163,8 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, f = vlib_get_frame (vm, p->frame); if (p->next_frame_index == VLIB_PENDING_FRAME_NO_NEXT_FRAME) { - /* No next frame: so use dummy on stack. */ - nf = &nf_dummy; + /* No next frame: so use placeholder on stack. */ + nf = &nf_placeholder; nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE; nf->frame = NULL; } @@ -1579,6 +1374,9 @@ dispatch_process (vlib_main_t * vm, old_process_index = nm->current_process_index; nm->current_process_index = node->runtime_index; + vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, + VLIB_NODE_RUNTIME_PERF_BEFORE); + n_vectors = vlib_process_startup (vm, p, f); nm->current_process_index = old_process_index; @@ -1618,6 +1416,9 @@ dispatch_process (vlib_main_t * vm, vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend, /* is_after */ 1); + vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, + VLIB_NODE_RUNTIME_PERF_AFTER); + vlib_process_update_stats (vm, p, /* n_calls */ !is_suspend, /* n_vectors */ n_vectors, @@ -1668,6 +1469,9 @@ dispatch_suspended_process (vlib_main_t * vm, /* Save away current process for suspend. */ nm->current_process_index = node->runtime_index; + vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, + VLIB_NODE_RUNTIME_PERF_BEFORE); + n_vectors = vlib_process_resume (vm, p); t = clib_cpu_time_now (); @@ -1701,6 +1505,9 @@ dispatch_suspended_process (vlib_main_t * vm, vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend, /* is_after */ 1); + vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, + VLIB_NODE_RUNTIME_PERF_AFTER); + vlib_process_update_stats (vm, p, /* n_calls */ !is_suspend, /* n_vectors */ n_vectors, @@ -1715,27 +1522,6 @@ vl_api_send_pending_rpc_requests (vlib_main_t * vm) { } -static_always_inline u64 -dispatch_pending_interrupts (vlib_main_t * vm, vlib_node_main_t * nm, - u64 cpu_time_now) -{ - vlib_node_runtime_t *n; - - for (int i = 0; i < _vec_len (nm->pending_local_interrupts); i++) - { - vlib_node_interrupt_t *in; - in = vec_elt_at_index (nm->pending_local_interrupts, i); - n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], - in->node_runtime_index); - n->interrupt_data = in->data; - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_INTERRUPT, /* frame */ 0, - cpu_time_now); - } - vec_reset_length (nm->pending_local_interrupts); - return cpu_time_now; -} - static_always_inline void vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { @@ -1764,11 +1550,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) cpu_time_now = clib_cpu_time_now (); /* Pre-allocate interupt runtime indices and lock. */ - vec_alloc (nm->pending_local_interrupts, 32); - vec_alloc (nm->pending_remote_interrupts, 32); - vec_alloc_aligned (nm->pending_remote_interrupts_notify, 1, - CLIB_CACHE_LINE_BYTES); - clib_spinlock_init (&nm->pending_interrupt_lock); + vec_alloc_aligned (nm->pending_interrupts, 1, CLIB_CACHE_LINE_BYTES); /* Pre-allocate expired nodes. */ if (!nm->polling_threshold_vector_length) @@ -1809,33 +1591,34 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } if (!is_main) + vlib_worker_thread_barrier_check (); + + if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter)) { - vlib_worker_thread_barrier_check (); - if (PREDICT_FALSE (vm->check_frame_queues + - frame_queue_check_counter)) - { - u32 processed = 0; + u32 processed = 0; - if (vm->check_frame_queues) - { - frame_queue_check_counter = 100; - vm->check_frame_queues = 0; - } + if (vm->check_frame_queues) + { + frame_queue_check_counter = 100; + vm->check_frame_queues = 0; + } - vec_foreach (fqm, tm->frame_queue_mains) - processed += vlib_frame_queue_dequeue (vm, fqm); + vec_foreach (fqm, tm->frame_queue_mains) + processed += vlib_frame_queue_dequeue (vm, fqm); - /* No handoff queue work found? */ - if (processed) - frame_queue_check_counter = 100; - else - frame_queue_check_counter--; - } - if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks))) - clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm); + /* No handoff queue work found? */ + if (processed) + frame_queue_check_counter = 100; + else + frame_queue_check_counter--; } + if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks))) + clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm, + cpu_time_now); + /* Process pre-input nodes. */ + cpu_time_now = clib_cpu_time_now (); vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT, @@ -1854,26 +1637,22 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) vm->queue_signal_callback (vm); - /* handle local interruots */ - if (_vec_len (nm->pending_local_interrupts)) - cpu_time_now = dispatch_pending_interrupts (vm, nm, cpu_time_now); - - /* handle remote interruots */ - if (PREDICT_FALSE (_vec_len (nm->pending_remote_interrupts))) + if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE)) { - vlib_node_interrupt_t *in; - - /* at this point it is known that - * vec_len (nm->pending_local_interrupts) is zero so we quickly swap - * local and remote vector under the spinlock */ - clib_spinlock_lock (&nm->pending_interrupt_lock); - in = nm->pending_local_interrupts; - nm->pending_local_interrupts = nm->pending_remote_interrupts; - nm->pending_remote_interrupts = in; - *nm->pending_remote_interrupts_notify = 0; - clib_spinlock_unlock (&nm->pending_interrupt_lock); - - cpu_time_now = dispatch_pending_interrupts (vm, nm, cpu_time_now); + int int_num = -1; + *nm->pending_interrupts = 0; + + while ((int_num = + clib_interrupt_get_next (nm->interrupts, int_num)) != -1) + { + vlib_node_runtime_t *n; + clib_interrupt_clear (nm->interrupts, int_num); + n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], + int_num); + cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, + VLIB_NODE_STATE_INTERRUPT, + /* frame */ 0, cpu_time_now); + } } /* Input nodes may have added work to the pending vector. @@ -2017,11 +1796,49 @@ vlib_worker_loop (vlib_main_t * vm) vlib_main_or_worker_loop (vm, /* is_main */ 0); } -vlib_main_t vlib_global_main; +vlib_global_main_t vlib_global_main; + +void +vlib_add_del_post_mortem_callback (void *cb, int is_add) +{ + vlib_global_main_t *vgm = vlib_get_global_main (); + int i; + + if (is_add == 0) + { + for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--) + if (vgm->post_mortem_callbacks[i] == cb) + vec_del1 (vgm->post_mortem_callbacks, i); + return; + } + + for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) + if (vgm->post_mortem_callbacks[i] == cb) + return; + vec_add1 (vgm->post_mortem_callbacks, cb); +} + +static void +elog_post_mortem_dump (void) +{ + elog_main_t *em = vlib_get_elog_main (); + + u8 *filename; + clib_error_t *error; + + filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0); + error = elog_write_file (em, (char *) filename, 1 /* flush ring */); + if (error) + clib_error_report (error); + /* + * We're in the middle of crashing. Don't try to free the filename. + */ +} static clib_error_t * vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) { + vlib_global_main_t *vgm = vlib_get_global_main (); int turn_on_mem_trace = 0; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) @@ -2030,10 +1847,12 @@ vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) turn_on_mem_trace = 1; else if (unformat (input, "elog-events %d", - &vm->elog_main.event_ring_size)) - ; + &vgm->configured_elog_ring_size)) + vgm->configured_elog_ring_size = + 1 << max_log2 (vgm->configured_elog_ring_size); else if (unformat (input, "elog-post-mortem-dump")) - vm->elog_post_mortem_dump = 1; + vlib_add_del_post_mortem_callback (elog_post_mortem_dump, + /* is_add */ 1); else if (unformat (input, "buffer-alloc-success-rate %f", &vm->buffer_alloc_success_rate)) { @@ -2064,7 +1883,7 @@ vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) VLIB_EARLY_CONFIG_FUNCTION (vlib_main_configure, "vlib"); static void -dummy_queue_signal_callback (vlib_main_t * vm) +placeholder_queue_signal_callback (vlib_main_t * vm) { } @@ -2107,22 +1926,22 @@ vl_api_get_elog_trace_api_messages (void) int vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) { + vlib_global_main_t *vgm = vlib_get_global_main (); clib_error_t *volatile error; vlib_node_main_t *nm = &vm->node_main; - vm->queue_signal_callback = dummy_queue_signal_callback; + vm->queue_signal_callback = placeholder_queue_signal_callback; - /* Turn on event log. */ - if (!vm->elog_main.event_ring_size) - vm->elog_main.event_ring_size = 128 << 10; - elog_init (&vm->elog_main, vm->elog_main.event_ring_size); - elog_enable_disable (&vm->elog_main, 1); - vl_api_set_elog_main (&vm->elog_main); + /* Reconfigure event log which is enabled very early */ + if (vgm->configured_elog_ring_size && + vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size) + elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size); + vl_api_set_elog_main (vlib_get_elog_main ()); (void) vl_api_set_elog_trace_api_messages (1); /* Default name. */ - if (!vm->name) - vm->name = "VLIB"; + if (!vgm->name) + vgm->name = "VLIB"; if ((error = vlib_physmem_init (vm))) { @@ -2148,6 +1967,9 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) goto done; } + /* Register node ifunction variants */ + vlib_register_all_node_march_variants (vm); + /* Register static nodes so that init functions may use them. */ vlib_register_all_static_nodes (vm); @@ -2187,8 +2009,8 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) } /* See unix/main.c; most likely already set up */ - if (vm->init_functions_called == 0) - vm->init_functions_called = hash_create (0, /* value bytes */ 0); + if (vgm->init_functions_called == 0) + vgm->init_functions_called = hash_create (0, /* value bytes */ 0); if ((error = vlib_call_all_init_functions (vm))) goto done; @@ -2230,7 +2052,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) vm->damping_constant = exp (-1.0 / 20.0); /* Sort per-thread init functions before we start threads */ - vlib_sort_init_exit_functions (&vm->worker_init_function_registrations); + vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations); /* Call all main loop enter functions. */ { @@ -2257,6 +2079,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) vlib_main_loop (vm); done: + vlib_worker_thread_barrier_sync (vm); /* Call all exit functions. */ { clib_error_t *sub_error; @@ -2264,6 +2087,7 @@ done: if (sub_error) clib_error_report (sub_error); } + vlib_worker_thread_barrier_release (vm); if (error) clib_error_report (error); @@ -2271,270 +2095,18 @@ done: return 0; } -int -vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t * a) +vlib_main_t * +vlib_get_main_not_inline (void) { - vlib_main_t *vm = vlib_get_main (); - pcap_main_t *pm = &vm->dispatch_pcap_main; - vlib_trace_main_t *tm; - vlib_trace_node_t *tn; - - if (a->status) - { - if (vm->dispatch_pcap_enable) - { - int i; - vlib_cli_output - (vm, "pcap dispatch capture enabled: %d of %d pkts...", - pm->n_packets_captured, pm->n_packets_to_capture); - vlib_cli_output (vm, "capture to file %s", pm->file_name); - - for (i = 0; i < vec_len (vm->dispatch_buffer_trace_nodes); i++) - { - vlib_cli_output (vm, - "Buffer trace of %d pkts from %U enabled...", - a->buffer_traces_to_capture, - format_vlib_node_name, vm, - vm->dispatch_buffer_trace_nodes[i]); - } - } - else - vlib_cli_output (vm, "pcap dispatch capture disabled"); - return 0; - } - - /* Consistency checks */ - - /* Enable w/ capture already enabled not allowed */ - if (vm->dispatch_pcap_enable && a->enable) - return -7; /* VNET_API_ERROR_INVALID_VALUE */ - - /* Disable capture with capture already disabled, not interesting */ - if (vm->dispatch_pcap_enable == 0 && a->enable == 0) - return -81; /* VNET_API_ERROR_VALUE_EXIST */ - - /* Change number of packets to capture while capturing */ - if (vm->dispatch_pcap_enable && a->enable - && (pm->n_packets_to_capture != a->packets_to_capture)) - return -8; /* VNET_API_ERROR_INVALID_VALUE_2 */ - - /* Independent of enable/disable, to allow buffer trace multi nodes */ - if (a->buffer_trace_node_index != ~0) - { - /* *INDENT-OFF* */ - foreach_vlib_main (( - { - tm = &this_vlib_main->trace_main; - tm->verbose = 0; /* not sure this ever did anything... */ - vec_validate (tm->nodes, a->buffer_trace_node_index); - tn = tm->nodes + a->buffer_trace_node_index; - tn->limit += a->buffer_traces_to_capture; - tm->trace_enable = 1; - })); - /* *INDENT-ON* */ - vec_add1 (vm->dispatch_buffer_trace_nodes, a->buffer_trace_node_index); - } - - if (a->enable) - { - /* Clean up from previous run, if any */ - vec_free (pm->file_name); - vec_free (pm->pcap_data); - memset (pm, 0, sizeof (*pm)); - - vec_validate_aligned (vnet_trace_dummy, 2048, CLIB_CACHE_LINE_BYTES); - if (pm->lock == 0) - clib_spinlock_init (&(pm->lock)); - - if (a->filename == 0) - a->filename = format (0, "/tmp/dispatch.pcap%c", 0); - - pm->file_name = (char *) a->filename; - pm->n_packets_captured = 0; - pm->packet_type = PCAP_PACKET_TYPE_vpp; - pm->n_packets_to_capture = a->packets_to_capture; - /* *INDENT-OFF* */ - foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 1;})); - /* *INDENT-ON* */ - } - else - { - /* *INDENT-OFF* */ - foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 0;})); - /* *INDENT-ON* */ - vec_reset_length (vm->dispatch_buffer_trace_nodes); - if (pm->n_packets_captured) - { - clib_error_t *error; - pm->n_packets_to_capture = pm->n_packets_captured; - vlib_cli_output (vm, "Write %d packets to %s, and stop capture...", - pm->n_packets_captured, pm->file_name); - error = pcap_write (pm); - if (pm->flags & PCAP_MAIN_INIT_DONE) - pcap_close (pm); - /* Report I/O errors... */ - if (error) - { - clib_error_report (error); - return -11; /* VNET_API_ERROR_SYSCALL_ERROR_1 */ - } - return 0; - } - else - return -6; /* VNET_API_ERROR_NO_SUCH_ENTRY */ - } - - return 0; + return vlib_get_main (); } -static clib_error_t * -dispatch_trace_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) +elog_main_t * +vlib_get_elog_main_not_inline () { - unformat_input_t _line_input, *line_input = &_line_input; - vlib_pcap_dispatch_trace_args_t _a, *a = &_a; - u8 *filename = 0; - u32 max = 1000; - int rv; - int enable = 0; - int status = 0; - u32 node_index = ~0, buffer_traces_to_capture = 100; - - /* Get a line of input. */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "on %=", &enable, 1)) - ; - else if (unformat (line_input, "enable %=", &enable, 1)) - ; - else if (unformat (line_input, "off %=", &enable, 0)) - ; - else if (unformat (line_input, "disable %=", &enable, 0)) - ; - else if (unformat (line_input, "max %d", &max)) - ; - else if (unformat (line_input, "packets-to-capture %d", &max)) - ; - else if (unformat (line_input, "file %U", unformat_vlib_tmpfile, - &filename)) - ; - else if (unformat (line_input, "status %=", &status, 1)) - ; - else if (unformat (line_input, "buffer-trace %U %d", - unformat_vlib_node, vm, &node_index, - &buffer_traces_to_capture)) - ; - else - { - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, line_input); - } - } - - unformat_free (line_input); - - /* no need for memset (a, 0, sizeof (*a)), set all fields here. */ - a->filename = filename; - a->enable = enable; - a->status = status; - a->packets_to_capture = max; - a->buffer_trace_node_index = node_index; - a->buffer_traces_to_capture = buffer_traces_to_capture; - - rv = vlib_pcap_dispatch_trace_configure (a); - - switch (rv) - { - case 0: - break; - - case -7: - return clib_error_return (0, "dispatch trace already enabled..."); - - case -81: - return clib_error_return (0, "dispatch trace already disabled..."); - - case -8: - return clib_error_return - (0, "can't change number of records to capture while tracing..."); - - case -11: - return clib_error_return (0, "I/O writing trace capture..."); - - case -6: - return clib_error_return (0, "No packets captured..."); - - default: - vlib_cli_output (vm, "WARNING: trace configure returned %d", rv); - break; - } - return 0; + return &vlib_global_main.elog_main; } -/*? - * This command is used to start or stop pcap dispatch trace capture, or show - * the capture status. - * - * This command has the following optional parameters: - * - * - on|off - Used to start or stop capture. - * - * - max - Depth of local buffer. Once 'nn' number - * of packets have been received, buffer is flushed to file. Once another - * 'nn' number of packets have been received, buffer is flushed - * to file, overwriting previous write. If not entered, value defaults - * to 100. Can only be updated if packet capture is off. - * - * - file - Used to specify the output filename. The file will - * be placed in the '/tmp' directory, so only the filename is - * supported. Directory should not be entered. If file already exists, file - * will be overwritten. If no filename is provided, '/tmp/vpe.pcap' - * will be used. Can only be updated if packet capture is off. - * - * - status - Displays the current status and configured attributes - * associated with a packet capture. If packet capture is in progress, - * 'status' also will return the number of packets currently in - * the local buffer. All additional attributes entered on command line - * with 'status' will be ignored and not applied. - * - * @cliexpar - * Example of how to display the status of capture when off: - * @cliexstart{pcap dispatch trace status} - * max is 100, for any interface to file /tmp/vpe.pcap - * pcap dispatch capture is off... - * @cliexend - * Example of how to start a dispatch trace capture: - * @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap} - * pcap dispatch capture on... - * @cliexend - * Example of how to start a dispatch trace capture with buffer tracing - * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000} - * pcap dispatch capture on... - * @cliexend - * Example of how to display the status of a tx packet capture in progress: - * @cliexstart{pcap tx trace status} - * max is 35, dispatch trace to file /tmp/vppTest.pcap - * pcap tx capture is on: 20 of 35 pkts... - * @cliexend - * Example of how to stop a tx packet capture: - * @cliexstart{vppctl pcap dispatch trace off} - * captured 21 pkts... - * saved to /tmp/dispatchTrace.pcap... - * @cliexend -?*/ -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (pcap_dispatch_trace_command, static) = { - .path = "pcap dispatch trace", - .short_help = - "pcap dispatch trace [on|off] [max ] [file ] [status]\n" - " [buffer-trace ]", - .function = dispatch_trace_command_fn, -}; -/* *INDENT-ON* */ - /* * fd.io coding-style-patch-verification: ON *