return p;
}
-static vlib_frame_size_t *
+static inline vlib_frame_size_t *
get_frame_size_info (vlib_node_main_t * nm,
u32 n_scalar_bytes, u32 n_vector_bytes)
{
+#ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
uword key = (n_scalar_bytes << 16) | n_vector_bytes;
uword *p, i;
}
return vec_elt_at_index (nm->frame_sizes, i);
+#else
+ ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes)
+ == (vlib_frame_bytes (0, 4)));
+ return vec_elt_at_index (nm->frame_sizes, 0);
+#endif
}
static u32
ASSERT (nf->frame_index != frame_index);
}
- f->frame_flags &= ~VLIB_FRAME_IS_ALLOCATED;
+ f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND);
vec_add1 (fs->free_frame_indices, frame_index);
ASSERT (fs->n_alloc_frames > 0);
{
nf->flags &= ~VLIB_FRAME_PENDING;
f->n_vectors = 0;
+ f->flags = 0;
}
- /* Allocate new frame if current one is already full. */
+ /* Allocate new frame if current one is marked as no-append or
+ it is already full. */
n_used = f->n_vectors;
- if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0))
+ if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
+ (f->frame_flags & VLIB_FRAME_NO_APPEND))
{
/* Old frame may need to be freed after dispatch, since we'll have
two redundant frames from node -> next node. */
vlib_frame_t *f;
u32 n_vectors_in_frame;
- if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0)
+ if (CLIB_DEBUG > 0)
vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
vlib_node_runtime_sync_stats (vlib_main_t * vm,
vlib_node_runtime_t * r,
uword n_calls, uword n_vectors, uword n_clocks,
- uword n_ticks)
+ uword n_ticks0, uword n_ticks1)
{
vlib_node_t *n = vlib_get_node (vm, r->node_index);
n->stats_total.calls += n_calls + r->calls_since_last_overflow;
n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
- n->stats_total.perf_counter_ticks += n_ticks +
- r->perf_counter_ticks_since_last_overflow;
+ n->stats_total.perf_counter0_ticks += n_ticks0 +
+ r->perf_counter0_ticks_since_last_overflow;
+ n->stats_total.perf_counter1_ticks += n_ticks1 +
+ r->perf_counter1_ticks_since_last_overflow;
n->stats_total.perf_counter_vectors += n_vectors +
r->perf_counter_vectors_since_last_overflow;
n->stats_total.max_clock = r->max_clock;
r->calls_since_last_overflow = 0;
r->vectors_since_last_overflow = 0;
r->clocks_since_last_overflow = 0;
- r->perf_counter_ticks_since_last_overflow = 0ULL;
+ r->perf_counter0_ticks_since_last_overflow = 0ULL;
+ r->perf_counter1_ticks_since_last_overflow = 0ULL;
r->perf_counter_vectors_since_last_overflow = 0ULL;
}
vlib_process_sync_stats (vlib_main_t * vm,
vlib_process_t * p,
uword n_calls, uword n_vectors, uword n_clocks,
- uword n_ticks)
+ uword n_ticks0, uword n_ticks1)
{
vlib_node_runtime_t *rt = &p->node_runtime;
vlib_node_t *n = vlib_get_node (vm, rt->node_index);
vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks,
- n_ticks);
+ n_ticks0, n_ticks1);
n->stats_total.suspends += p->n_suspends;
p->n_suspends = 0;
}
vec_elt_at_index (vm->node_main.nodes_by_type[n->type],
n->runtime_index);
- vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0);
+ vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0, 0);
/* Sync up runtime next frame vector counters with main node structure. */
{
vlib_node_runtime_t * node,
uword n_calls,
uword n_vectors, uword n_clocks,
- uword n_ticks)
+ uword n_ticks0, uword n_ticks1)
{
u32 ca0, ca1, v0, v1, cl0, cl1, r;
- u32 ptick0, ptick1, pvec0, pvec1;
+ u32 ptick00, ptick01, ptick10, ptick11, pvec0, pvec1;
cl0 = cl1 = node->clocks_since_last_overflow;
ca0 = ca1 = node->calls_since_last_overflow;
v0 = v1 = node->vectors_since_last_overflow;
- ptick0 = ptick1 = node->perf_counter_ticks_since_last_overflow;
+ ptick00 = ptick01 = node->perf_counter0_ticks_since_last_overflow;
+ ptick10 = ptick11 = node->perf_counter1_ticks_since_last_overflow;
pvec0 = pvec1 = node->perf_counter_vectors_since_last_overflow;
ca1 = ca0 + n_calls;
v1 = v0 + n_vectors;
cl1 = cl0 + n_clocks;
- ptick1 = ptick0 + n_ticks;
+ ptick01 = ptick00 + n_ticks0;
+ ptick11 = ptick10 + n_ticks1;
pvec1 = pvec0 + n_vectors;
node->calls_since_last_overflow = ca1;
node->clocks_since_last_overflow = cl1;
node->vectors_since_last_overflow = v1;
- node->perf_counter_ticks_since_last_overflow = ptick1;
+ node->perf_counter0_ticks_since_last_overflow = ptick01;
+ node->perf_counter1_ticks_since_last_overflow = ptick11;
node->perf_counter_vectors_since_last_overflow = pvec1;
node->max_clock_n = node->max_clock > n_clocks ?
r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
- if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick1 < ptick0)
- || (pvec1 < pvec0))
+ if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick01 < ptick00)
+ || (ptick11 < ptick10) || (pvec1 < pvec0))
{
node->calls_since_last_overflow = ca0;
node->clocks_since_last_overflow = cl0;
node->vectors_since_last_overflow = v0;
- node->perf_counter_ticks_since_last_overflow = ptick0;
+ node->perf_counter0_ticks_since_last_overflow = ptick00;
+ node->perf_counter1_ticks_since_last_overflow = ptick10;
node->perf_counter_vectors_since_last_overflow = pvec0;
vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks,
- n_ticks);
+ n_ticks0, n_ticks1);
}
return r;
}
-static inline u64
-vlib_node_runtime_perf_counter (vlib_main_t * vm)
+always_inline void
+vlib_node_runtime_perf_counter (vlib_main_t * vm, u64 * pmc0, u64 * pmc1,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame, int before_or_after)
{
+ *pmc0 = 0;
+ *pmc1 = 0;
if (PREDICT_FALSE (vm->vlib_node_runtime_perf_counter_cb != 0))
- return ((*vm->vlib_node_runtime_perf_counter_cb) (vm));
- return 0ULL;
+ (*vm->vlib_node_runtime_perf_counter_cb) (vm, pmc0, pmc1, node,
+ frame, before_or_after);
}
always_inline void
vlib_process_update_stats (vlib_main_t * vm,
vlib_process_t * p,
- uword n_calls, uword n_vectors, uword n_clocks,
- uword n_ticks)
+ uword n_calls, uword n_vectors, uword n_clocks)
{
vlib_node_runtime_update_stats (vm, &p->node_runtime,
- n_calls, n_vectors, n_clocks, n_ticks);
+ n_calls, n_vectors, n_clocks, 0ULL, 0ULL);
}
static clib_error_t *
{
vlib_main_t *evm = &vlib_global_main;
elog_main_t *em = &evm->elog_main;
+ int enabled = evm->elog_trace_graph_dispatch |
+ evm->elog_trace_graph_circuit;
- if (VLIB_ELOG_MAIN_LOOP && n_vectors)
- elog_track (em,
- /* event type */
- vec_elt_at_index (is_return
- ? evm->node_return_elog_event_types
- : evm->node_call_elog_event_types,
- node_index),
- /* track */
- (vm->thread_index ? &vlib_worker_threads[vm->thread_index].
- elog_track : &em->default_track),
- /* data to log */ n_vectors);
+ if (PREDICT_FALSE (enabled && n_vectors))
+ {
+ if (PREDICT_FALSE (!elog_is_enabled (em)))
+ {
+ evm->elog_trace_graph_dispatch = 0;
+ evm->elog_trace_graph_circuit = 0;
+ return;
+ }
+ if (PREDICT_TRUE
+ (evm->elog_trace_graph_dispatch ||
+ (evm->elog_trace_graph_circuit &&
+ node_index == evm->elog_trace_graph_circuit_node_index)))
+ {
+ elog_track (em,
+ /* event type */
+ vec_elt_at_index (is_return
+ ? evm->node_return_elog_event_types
+ : evm->node_call_elog_event_types,
+ node_index),
+ /* track */
+ (vm->thread_index ?
+ &vlib_worker_threads[vm->thread_index].elog_track
+ : &em->default_track),
+ /* data to log */ n_vectors);
+ }
+ }
}
#if VLIB_BUFFER_TRACE_TRAJECTORY > 0
#endif
}
+u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak));
+u8 *
+format_vnet_buffer_flags (u8 * s, va_list * args)
+{
+ s = format (s, "BUG STUB %s", __FUNCTION__);
+ return s;
+}
+
+u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak));
+u8 *
+format_vnet_buffer_opaque (u8 * s, va_list * args)
+{
+ s = format (s, "BUG STUB %s", __FUNCTION__);
+ return s;
+}
+
+u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args)
+ __attribute__ ((weak));
+u8 *
+format_vnet_buffer_opaque2 (u8 * s, va_list * args)
+{
+ s = format (s, "BUG STUB %s", __FUNCTION__);
+ return s;
+}
+
+static u8 *
+format_buffer_metadata (u8 * s, va_list * args)
+{
+ vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+
+ s = format (s, "flags: %U\n", format_vnet_buffer_flags, b);
+ s = format (s, "current_data: %d, current_length: %d\n",
+ (i32) (b->current_data), (i32) (b->current_length));
+ s = format (s, "current_config_index: %d, flow_id: %x, next_buffer: %x\n",
+ b->current_config_index, b->flow_id, b->next_buffer);
+ s = format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
+ (u32) (b->error), (u32) (b->ref_count),
+ (u32) (b->buffer_pool_index));
+ s = format (s,
+ "trace_index: %d, len_not_first_buf: %d\n",
+ b->trace_index, b->total_length_not_including_first_buffer);
+ return s;
+}
+
+#define A(x) vec_add1(vm->pcap_buffer, (x))
+
static void
dispatch_pcap_trace (vlib_main_t * vm,
vlib_node_runtime_t * node, vlib_frame_t * frame)
{
int i;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b;
- u8 name_tlv[64];
pcap_main_t *pm = &vm->dispatch_pcap_main;
+ vlib_trace_main_t *tm = &vm->trace_main;
u32 capture_size;
vlib_node_t *n;
i32 n_left;
f64 time_now = vlib_time_now (vm);
u32 *from;
- u32 name_length;
u8 *d;
+ u8 string_count;
/* Input nodes don't have frames yet */
if (frame == 0 || frame->n_vectors == 0)
vlib_get_buffers (vm, from, bufs, frame->n_vectors);
bufp = bufs;
- /* Create a node name TLV, since WS can't possibly guess */
n = vlib_get_node (vm, node->node_index);
- name_length = vec_len (n->name);
- name_length = name_length < ARRAY_LEN (name_tlv) - 2 ?
- name_length : ARRAY_LEN (name_tlv) - 2;
-
- name_tlv[0] = (u8) name_length;
- clib_memcpy_fast (name_tlv + 1, n->name, name_length);
- name_tlv[name_length + 1] = 0;
for (i = 0; i < frame->n_vectors; i++)
{
{
b = bufp[i];
- /* Figure out how many bytes we're capturing */
- capture_size = (sizeof (vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE) + vlib_buffer_length_in_chain (vm, b) + sizeof (u32) + +(name_length + 2); /* +2: count plus NULL byte */
+ vec_reset_length (vm->pcap_buffer);
+ string_count = 0;
+
+ /* Version, flags */
+ A ((u8) VLIB_PCAP_MAJOR_VERSION);
+ A ((u8) VLIB_PCAP_MINOR_VERSION);
+ A (0 /* string_count */ );
+ A (n->protocol_hint);
+
+ /* Buffer index (big endian) */
+ A ((from[i] >> 24) & 0xff);
+ A ((from[i] >> 16) & 0xff);
+ A ((from[i] >> 8) & 0xff);
+ A ((from[i] >> 0) & 0xff);
+
+ /* Node name, NULL-terminated ASCII */
+ vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0);
+ string_count++;
+
+ vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
+ format_buffer_metadata, b, 0);
+ string_count++;
+ vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
+ format_vnet_buffer_opaque, b, 0);
+ string_count++;
+ vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
+ format_vnet_buffer_opaque2, b, 0);
+ string_count++;
+
+ /* Is this packet traced? */
+ if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ vlib_trace_header_t **h
+ = pool_elt_at_index (tm->trace_buffer_pool, b->trace_index);
- clib_spinlock_lock_if_init (&pm->lock);
- n_left = clib_min (capture_size, 512);
- d = pcap_add_packet (pm, time_now, n_left, capture_size);
+ vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
+ format_vlib_trace, vm, h[0], 0);
+ string_count++;
+ }
- /* Copy the buffer index */
- clib_memcpy_fast (d, &from[i], sizeof (u32));
- d += 4;
+ /* Save the string count */
+ vm->pcap_buffer[2] = string_count;
- /* Copy the name TLV */
- clib_memcpy_fast (d, name_tlv, name_length + 2);
- d += name_length + 2;
+ /* Figure out how many bytes in the pcap trace */
+ capture_size = vec_len (vm->pcap_buffer) +
+ +vlib_buffer_length_in_chain (vm, b);
+
+ clib_spinlock_lock_if_init (&pm->lock);
+ n_left = clib_min (capture_size, 16384);
+ d = pcap_add_packet (pm, time_now, n_left, capture_size);
- /* Copy the buffer metadata, but not the rewrite space */
- clib_memcpy_fast (d, b, sizeof (*b) - VLIB_BUFFER_PRE_DATA_SIZE);
- d += sizeof (*b) - VLIB_BUFFER_PRE_DATA_SIZE;
+ /* Copy the header */
+ clib_memcpy_fast (d, vm->pcap_buffer, vec_len (vm->pcap_buffer));
+ d += vec_len (vm->pcap_buffer);
- n_left = clib_min (vlib_buffer_length_in_chain (vm, b),
- 512 - (sizeof (*b) - VLIB_BUFFER_PRE_DATA_SIZE));
+ n_left = clib_min
+ (vlib_buffer_length_in_chain (vm, b),
+ (16384 - vec_len (vm->pcap_buffer)));
/* Copy the packet data */
while (1)
{
u64 t;
vlib_node_main_t *nm = &vm->node_main;
vlib_next_frame_t *nf;
+ u64 pmc_before[2], pmc_after[2], pmc_delta[2];
if (CLIB_DEBUG > 0)
{
vm->cpu_time_last_node_dispatch = last_time_stamp;
- if (1 /* || vm->thread_index == node->thread_index */ )
- {
- u64 pmc_before, pmc_delta;
+ vlib_elog_main_loop_event (vm, node->node_index,
+ last_time_stamp, frame ? frame->n_vectors : 0,
+ /* is_after */ 0);
- vlib_elog_main_loop_event (vm, node->node_index,
- last_time_stamp,
- frame ? frame->n_vectors : 0,
- /* is_after */ 0);
+ vlib_node_runtime_perf_counter (vm, &pmc_before[0], &pmc_before[1],
+ node, frame, 0 /* before */ );
- /*
- * To validate accounting: pmc_before = last_time_stamp
- * perf ticks should equal clocks/pkt...
- */
- pmc_before = vlib_node_runtime_perf_counter (vm);
-
- /*
- * Turn this on if you run into
- * "bad monkey" contexts, and you want to know exactly
- * which nodes they've visited... See ixge.c...
- */
- if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
- {
- int i;
- u32 *from;
- from = vlib_frame_vector_args (frame);
- for (i = 0; i < frame->n_vectors; i++)
- {
- vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
- add_trajectory_trace (b, node->node_index);
- }
- if (PREDICT_FALSE (vm->dispatch_pcap_enable))
- dispatch_pcap_trace (vm, node, frame);
- n = node->function (vm, node, frame);
- }
- else
+ /*
+ * Turn this on if you run into
+ * "bad monkey" contexts, and you want to know exactly
+ * which nodes they've visited... See ixge.c...
+ */
+ if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
+ {
+ int i;
+ u32 *from;
+ from = vlib_frame_vector_args (frame);
+ for (i = 0; i < frame->n_vectors; i++)
{
- if (PREDICT_FALSE (vm->dispatch_pcap_enable))
- dispatch_pcap_trace (vm, node, frame);
- n = node->function (vm, node, frame);
+ vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
+ add_trajectory_trace (b, node->node_index);
}
+ if (PREDICT_FALSE (vm->dispatch_pcap_enable))
+ dispatch_pcap_trace (vm, node, frame);
+ n = node->function (vm, node, frame);
+ }
+ else
+ {
+ if (PREDICT_FALSE (vm->dispatch_pcap_enable))
+ dispatch_pcap_trace (vm, node, frame);
+ n = node->function (vm, node, frame);
+ }
- t = clib_cpu_time_now ();
+ t = clib_cpu_time_now ();
- /*
- * To validate accounting: pmc_delta = t - pmc_before;
- * perf ticks should equal clocks/pkt...
- */
- pmc_delta = vlib_node_runtime_perf_counter (vm) - pmc_before;
-
- vlib_elog_main_loop_event (vm, node->node_index, t, n, /* is_after */
- 1);
-
- vm->main_loop_vectors_processed += n;
- vm->main_loop_nodes_processed += n > 0;
-
- v = vlib_node_runtime_update_stats (vm, node,
- /* n_calls */ 1,
- /* n_vectors */ n,
- /* n_clocks */ t - last_time_stamp,
- pmc_delta /* PMC ticks */ );
-
- /* When in interrupt mode and vector rate crosses threshold switch to
- polling mode. */
- if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
- || (dispatch_state == VLIB_NODE_STATE_POLLING
- && (node->flags
- & VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)))
- {
-#ifdef DISPATCH_NODE_ELOG_REQUIRED
- ELOG_TYPE_DECLARE (e) =
- {
- .function = (char *) __FUNCTION__,.format =
- "%s vector length %d, switching to %s",.format_args =
- "T4i4t4",.n_enum_strings = 2,.enum_strings =
- {
- "interrupt", "polling",},};
- struct
- {
- u32 node_name, vector_length, is_polling;
- } *ed;
- vlib_worker_thread_t *w = vlib_worker_threads + vm->thread_index;
-#endif
+ /*
+ * To validate accounting: pmc_delta = t - pmc_before;
+ * perf ticks should equal clocks/pkt...
+ */
+ vlib_node_runtime_perf_counter (vm, &pmc_after[0], &pmc_after[1], node,
+ frame, 1 /* after */ );
+
+ pmc_delta[0] = pmc_after[0] - pmc_before[0];
+ pmc_delta[1] = pmc_after[1] - pmc_before[1];
+
+ vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
+
+ vm->main_loop_vectors_processed += n;
+ vm->main_loop_nodes_processed += n > 0;
+
+ v = vlib_node_runtime_update_stats (vm, node,
+ /* n_calls */ 1,
+ /* n_vectors */ n,
+ /* n_clocks */ t - last_time_stamp,
+ pmc_delta[0] /* PMC0 */ ,
+ pmc_delta[1] /* PMC1 */ );
+
+ /* When in interrupt mode and vector rate crosses threshold switch to
+ polling mode. */
+ if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
+ || (dispatch_state == VLIB_NODE_STATE_POLLING
+ && (node->flags
+ &
+ VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))))
+ {
+ /* *INDENT-OFF* */
+ ELOG_TYPE_DECLARE (e) =
+ {
+ .function = (char *) __FUNCTION__,
+ .format = "%s vector length %d, switching to %s",
+ .format_args = "T4i4t4",
+ .n_enum_strings = 2,
+ .enum_strings = {
+ "interrupt", "polling",
+ },
+ };
+ /* *INDENT-ON* */
+ struct
+ {
+ u32 node_name, vector_length, is_polling;
+ } *ed;
- if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
- && v >= nm->polling_threshold_vector_length) &&
- !(node->flags &
- VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
+ if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
+ && v >= nm->polling_threshold_vector_length) &&
+ !(node->flags &
+ VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))
+ {
+ vlib_node_t *n = vlib_get_node (vm, node->node_index);
+ n->state = VLIB_NODE_STATE_POLLING;
+ node->state = VLIB_NODE_STATE_POLLING;
+ node->flags &=
+ ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
+ node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
+ nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
+ nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
+
+ if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
{
- vlib_node_t *n = vlib_get_node (vm, node->node_index);
- n->state = VLIB_NODE_STATE_POLLING;
- node->state = VLIB_NODE_STATE_POLLING;
- node->flags &=
- ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
- node->flags |=
- VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
- nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
- nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
+ vlib_worker_thread_t *w = vlib_worker_threads
+ + vm->thread_index;
-#ifdef DISPATCH_NODE_ELOG_REQUIRED
ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
w->elog_track);
ed->node_name = n->name_elog_string;
ed->vector_length = v;
ed->is_polling = 1;
-#endif
}
- else if (dispatch_state == VLIB_NODE_STATE_POLLING
- && v <= nm->interrupt_threshold_vector_length)
+ }
+ else if (dispatch_state == VLIB_NODE_STATE_POLLING
+ && v <= nm->interrupt_threshold_vector_length)
+ {
+ vlib_node_t *n = vlib_get_node (vm, node->node_index);
+ if (node->flags &
+ VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
{
- vlib_node_t *n = vlib_get_node (vm, node->node_index);
- if (node->flags &
- VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE)
- {
- /* Switch to interrupt mode after dispatch in polling one more time.
- This allows driver to re-enable interrupts. */
- n->state = VLIB_NODE_STATE_INTERRUPT;
- node->state = VLIB_NODE_STATE_INTERRUPT;
- node->flags &=
- ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
- nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -=
- 1;
- nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] +=
- 1;
+ /* Switch to interrupt mode after dispatch in polling one more time.
+ This allows driver to re-enable interrupts. */
+ n->state = VLIB_NODE_STATE_INTERRUPT;
+ node->state = VLIB_NODE_STATE_INTERRUPT;
+ node->flags &=
+ ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE;
+ nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
+ nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
- }
- else
+ }
+ else
+ {
+ vlib_worker_thread_t *w = vlib_worker_threads
+ + vm->thread_index;
+ node->flags |=
+ VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
+ if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
{
- node->flags |=
- VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
-#ifdef DISPATCH_NODE_ELOG_REQUIRED
ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
w->elog_track);
ed->node_name = n->name_elog_string;
ed->vector_length = v;
ed->is_polling = 0;
-#endif
}
}
}
VLIB_NODE_STATE_POLLING,
f, last_time_stamp);
- f->frame_flags &= ~VLIB_FRAME_PENDING;
+ f->frame_flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_NO_APPEND);
/* Frame is ready to be used again, so restore it. */
if (restore_frame_index != ~0)
vlib_process_update_stats (vm, p,
/* n_calls */ !is_suspend,
/* n_vectors */ n_vectors,
- /* n_clocks */ t - last_time_stamp,
- /* pmc_ticks */ 0ULL);
+ /* n_clocks */ t - last_time_stamp);
return t;
}
vlib_process_update_stats (vm, p,
/* n_calls */ !is_suspend,
/* n_vectors */ n_vectors,
- /* n_clocks */ t - last_time_stamp,
- /* pmc_ticks */ 0ULL);
+ /* n_clocks */ t - last_time_stamp);
return t;
}
u64 cpu_time_now;
vlib_frame_queue_main_t *fqm;
u32 *last_node_runtime_indices = 0;
+ u32 frame_queue_check_counter = 0;
/* Initialize pending node vector. */
if (is_main)
if (!nm->interrupt_threshold_vector_length)
nm->interrupt_threshold_vector_length = 5;
- /* Make sure the performance monitor counter is disabled */
- vm->perf_counter_id = ~0;
+ vm->cpu_id = clib_get_current_cpu_id ();
+ vm->numa_node = clib_get_current_numa_node ();
/* Start all processes. */
if (is_main)
if (!is_main)
{
vlib_worker_thread_barrier_check ();
- vec_foreach (fqm, tm->frame_queue_mains)
- vlib_frame_queue_dequeue (vm, fqm);
+ if (PREDICT_FALSE (vm->check_frame_queues +
+ frame_queue_check_counter))
+ {
+ u32 processed = 0;
+
+ if (vm->check_frame_queues)
+ {
+ frame_queue_check_counter = 100;
+ vm->check_frame_queues = 0;
+ }
+
+ vec_foreach (fqm, tm->frame_queue_mains)
+ processed += vlib_frame_queue_dequeue (vm, fqm);
+
+ /* No handoff queue work found? */
+ if (processed)
+ frame_queue_check_counter = 100;
+ else
+ frame_queue_check_counter--;
+ }
if (PREDICT_FALSE (vm->worker_thread_main_loop_callback != 0))
((void (*)(vlib_main_t *)) vm->worker_thread_main_loop_callback)
(vm);
if (is_main)
{
+ /* *INDENT-OFF* */
+ ELOG_TYPE_DECLARE (es) =
+ {
+ .format = "process tw start",
+ .format_args = "",
+ };
+ ELOG_TYPE_DECLARE (ee) =
+ {
+ .format = "process tw end: %d",
+ .format_args = "i4",
+ };
+ /* *INDENT-ON* */
+
+ struct
+ {
+ int nready_procs;
+ } *ed;
+
/* Check if process nodes have expired from timing wheel. */
ASSERT (nm->data_from_advancing_timing_wheel != 0);
+ if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
+ ed = ELOG_DATA (&vlib_global_main.elog_main, es);
+
nm->data_from_advancing_timing_wheel =
TW (tw_timer_expire_timers_vec)
((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
ASSERT (nm->data_from_advancing_timing_wheel != 0);
+ if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
+ {
+ ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
+ ed->nready_procs =
+ _vec_len (nm->data_from_advancing_timing_wheel);
+ }
+
if (PREDICT_FALSE
(_vec_len (nm->data_from_advancing_timing_wheel) > 0))
{
goto done;
}
- if ((error = vlib_buffer_main_init (vm)))
+ if ((error = vlib_map_stat_segment_init (vm)))
{
clib_error_report (error);
goto done;
}
- if ((error = vlib_thread_init (vm)))
+ if ((error = vlib_buffer_main_init (vm)))
{
clib_error_report (error);
goto done;
}
- if ((error = vlib_map_stat_segment_init (vm)))
+ if ((error = vlib_thread_init (vm)))
{
clib_error_report (error);
goto done;
if ((error = vlib_call_all_init_functions (vm)))
goto done;
- /* Create default buffer free list. */
- vlib_buffer_create_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
- "default");
-
nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
CLIB_CACHE_LINE_BYTES);
vec_validate (vm->processing_rpc_requests, 0);
_vec_len (vm->processing_rpc_requests) = 0;
+ if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
+ goto done;
+
+ /* Call all main loop enter functions. */
+ {
+ clib_error_t *sub_error;
+ sub_error = vlib_call_all_main_loop_enter_functions (vm);
+ if (sub_error)
+ clib_error_report (sub_error);
+ }
+
switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE))
{
case VLIB_MAIN_LOOP_EXIT_NONE:
goto done;
}
- if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
- goto done;
-
- /* Call all main loop enter functions. */
- {
- clib_error_t *sub_error;
- sub_error = vlib_call_all_main_loop_enter_functions (vm);
- if (sub_error)
- clib_error_report (sub_error);
- }
-
vlib_main_loop (vm);
done:
unformat_input_t * input,
vlib_cli_command_t * cmd, int rx_tx)
{
-#define PCAP_DEF_PKT_TO_CAPTURE (100)
-
unformat_input_t _line_input, *line_input = &_line_input;
pcap_main_t *pm = &vm->dispatch_pcap_main;
- u8 *filename;
- u8 *chroot_filename = 0;
- u32 max = 0;
+ u8 *filename = 0;
+ u32 max = 1000;
int enabled = 0;
- int errorFlag = 0;
+ int is_error = 0;
clib_error_t *error = 0;
+ u32 node_index, add;
+ vlib_trace_main_t *tm;
+ vlib_trace_node_t *tn;
/* Get a line of input. */
if (!unformat_user (input, unformat_line_input, line_input))
else
{
vlib_cli_output (vm, "pcap dispatch capture already on...");
- errorFlag = 1;
+ is_error = 1;
break;
}
}
else
{
vlib_cli_output (vm, "pcap tx capture already off...");
- errorFlag = 1;
+ is_error = 1;
break;
}
}
vlib_cli_output
(vm,
"can't change max value while pcap tx capture active...");
- errorFlag = 1;
+ is_error = 1;
break;
}
pm->n_packets_to_capture = max;
}
- else if (unformat (line_input, "file %s", &filename))
+ else
+ if (unformat
+ (line_input, "file %U", unformat_vlib_tmpfile, &filename))
{
if (vm->dispatch_pcap_enable)
{
vlib_cli_output
(vm, "can't change file while pcap tx capture active...");
- errorFlag = 1;
- break;
- }
-
- /* Brain-police user path input */
- if (strstr ((char *) filename, "..")
- || index ((char *) filename, '/'))
- {
- vlib_cli_output (vm, "illegal characters in filename '%s'",
- filename);
- vlib_cli_output (vm, "Hint: .. and / are not allowed.");
- vec_free (filename);
- errorFlag = 1;
+ is_error = 1;
break;
}
-
- chroot_filename = format (0, "/tmp/%s%c", filename, 0);
- vec_free (filename);
}
else if (unformat (line_input, "status"))
{
}
break;
}
+ else if (unformat (line_input, "buffer-trace %U %d",
+ unformat_vlib_node, vm, &node_index, &add))
+ {
+ if (vnet_trace_dummy == 0)
+ vec_validate_aligned (vnet_trace_dummy, 2048,
+ CLIB_CACHE_LINE_BYTES);
+ vlib_cli_output (vm, "Buffer tracing of %d pkts from %U enabled...",
+ add, format_vlib_node_name, vm, node_index);
+
+ /* *INDENT-OFF* */
+ foreach_vlib_main ((
+ {
+ tm = &this_vlib_main->trace_main;
+ tm->verbose = 0; /* not sure this ever did anything... */
+ vec_validate (tm->nodes, node_index);
+ tn = tm->nodes + node_index;
+ tn->limit += add;
+ tm->trace_enable = 1;
+ }));
+ /* *INDENT-ON* */
+ }
else
{
error = clib_error_return (0, "unknown input `%U'",
format_unformat_error, line_input);
- errorFlag = 1;
+ is_error = 1;
break;
}
}
unformat_free (line_input);
-
- if (errorFlag == 0)
+ if (is_error == 0)
{
- /* Since no error, save configured values. */
- if (chroot_filename)
- {
- if (pm->file_name)
- vec_free (pm->file_name);
- vec_add1 (chroot_filename, 0);
- pm->file_name = (char *) chroot_filename;
- }
+ /* Clean up from previous run */
+ vec_free (pm->file_name);
+ vec_free (pm->pcap_data);
- if (max)
- pm->n_packets_to_capture = max;
+ memset (pm, 0, sizeof (*pm));
+ pm->n_packets_to_capture = max;
if (enabled)
{
- if (pm->file_name == 0)
- pm->file_name = (char *) format (0, "/tmp/dispatch.pcap%c", 0);
+ if (filename == 0)
+ filename = format (0, "/tmp/dispatch.pcap%c", 0);
+ pm->file_name = (char *) filename;
pm->n_packets_captured = 0;
- pm->packet_type = PCAP_PACKET_TYPE_user13;
+ pm->packet_type = PCAP_PACKET_TYPE_vpp;
if (pm->lock == 0)
clib_spinlock_init (&(pm->lock));
vm->dispatch_pcap_enable = 1;
vlib_cli_output (vm, "pcap dispatch capture on...");
}
}
- else if (chroot_filename)
- vec_free (chroot_filename);
return error;
}
* @cliexend
* Example of how to start a dispatch trace capture:
* @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap}
- * pcap dispatc capture on...
+ * pcap dispatch capture on...
+ * @cliexend
+ * Example of how to start a dispatch trace capture with buffer tracing
+ * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000}
+ * pcap dispatch capture on...
* @cliexend
* Example of how to display the status of a tx packet capture in progress:
* @cliexstart{pcap tx trace status}
VLIB_CLI_COMMAND (pcap_dispatch_trace_command, static) = {
.path = "pcap dispatch trace",
.short_help =
- "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]",
+ "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]\n"
+ " [buffer-trace <input-node-name> <nn>]",
.function = pcap_dispatch_trace_command_fn,
};
/* *INDENT-ON* */
Code Review / vpp.git / blobdiff