X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fmain.c;h=62599437f35c21b145980b4f36e69a5cfa4e038c;hb=98afc711c517ee860a2259ae18d496abd3416ba6;hp=73548fbea72834851acae9fc5054e665cfaae82c;hpb=04a7f05e91e919f51eaecaee476435484076655b;p=vpp.git diff --git a/src/vlib/main.c b/src/vlib/main.c index 73548fbea72..62599437f35 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -138,19 +138,12 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, else { f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN); - f->thread_index = vm->thread_index; fi = vlib_frame_index_no_check (vm, f); } /* Poison frame when debugging. */ if (CLIB_DEBUG > 0) - { - u32 save_thread_index = f->thread_index; - - memset (f, 0xfe, n); - - f->thread_index = save_thread_index; - } + memset (f, 0xfe, n); /* Insert magic number. */ { @@ -160,7 +153,7 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, *magic = VLIB_FRAME_MAGIC; } - f->flags = VLIB_FRAME_IS_ALLOCATED | frame_flags; + f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags; f->n_vectors = 0; f->scalar_size = scalar_size; f->vector_size = vector_size; @@ -207,7 +200,7 @@ vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f) vec_add2 (vm->node_main.pending_frames, p, 1); - f->flags |= VLIB_FRAME_PENDING; + f->frame_flags |= VLIB_FRAME_PENDING; p->frame_index = vlib_frame_index (vm, f); p->node_runtime_index = to_node->runtime_index; p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME; @@ -222,14 +215,14 @@ vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f) vlib_frame_size_t *fs; u32 frame_index; - ASSERT (f->flags & VLIB_FRAME_IS_ALLOCATED); + ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); node = vlib_get_node (vm, r->node_index); fs = get_frame_size_info (nm, node->scalar_size, node->vector_size); frame_index = vlib_frame_index (vm, f); - ASSERT (f->flags & VLIB_FRAME_IS_ALLOCATED); + ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); /* No next frames may point to freed frame. */ if (CLIB_DEBUG > 0) @@ -239,7 +232,7 @@ vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f) ASSERT (nf->frame_index != frame_index); } - f->flags &= ~VLIB_FRAME_IS_ALLOCATED; + f->frame_flags &= ~VLIB_FRAME_IS_ALLOCATED; vec_add1 (fs->free_frame_indices, frame_index); ASSERT (fs->n_alloc_frames > 0); @@ -385,7 +378,8 @@ vlib_get_next_frame_internal (vlib_main_t * vm, /* Has frame been removed from pending vector (e.g. finished dispatching)? If so we can reuse frame. */ - if ((nf->flags & VLIB_FRAME_PENDING) && !(f->flags & VLIB_FRAME_PENDING)) + if ((nf->flags & VLIB_FRAME_PENDING) + && !(f->frame_flags & VLIB_FRAME_PENDING)) { nf->flags &= ~VLIB_FRAME_PENDING; f->n_vectors = 0; @@ -400,7 +394,7 @@ vlib_get_next_frame_internal (vlib_main_t * vm, if (!(nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH)) { vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index); - f_old->flags |= VLIB_FRAME_FREE_AFTER_DISPATCH; + f_old->frame_flags |= VLIB_FRAME_FREE_AFTER_DISPATCH; } /* Allocate new frame to replace full one. */ @@ -467,7 +461,7 @@ vlib_put_next_frame (vlib_main_t * vm, vlib_frame_t *f; u32 n_vectors_in_frame; - if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0) + if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0) vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); nf = vlib_node_runtime_get_next_frame (vm, r, next_index); @@ -495,7 +489,7 @@ vlib_put_next_frame (vlib_main_t * vm, r->cached_next_index = next_index; - if (!(f->flags & VLIB_FRAME_PENDING)) + if (!(f->frame_flags & VLIB_FRAME_PENDING)) { __attribute__ ((unused)) vlib_node_t *node; vlib_node_t *next_node; @@ -511,7 +505,7 @@ vlib_put_next_frame (vlib_main_t * vm, p->node_runtime_index = nf->node_runtime_index; p->next_frame_index = nf - nm->next_frames; nf->flags |= VLIB_FRAME_PENDING; - f->flags |= VLIB_FRAME_PENDING; + f->frame_flags |= VLIB_FRAME_PENDING; /* * If we're going to dispatch this frame on another thread, @@ -891,52 +885,32 @@ vlib_elog_main_loop_event (vlib_main_t * vm, /* data to log */ n_vectors); } +#if VLIB_BUFFER_TRACE_TRAJECTORY > 0 +void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index); +void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b); + void -vlib_dump_context_trace (vlib_main_t * vm, u32 bi) +vlib_buffer_trace_trajectory_init (vlib_buffer_t * b) { - vlib_node_main_t *vnm = &vm->node_main; - vlib_buffer_t *b; - u8 i, n; - - if (VLIB_BUFFER_TRACE_TRAJECTORY) + if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0)) { - b = vlib_get_buffer (vm, bi); - n = b->pre_data[0]; - - fformat (stderr, "Context trace for bi %d b 0x%llx, visited %d\n", - bi, b, n); - - if (n == 0 || n > 20) - { - fformat (stderr, "n is unreasonable\n"); - return; - } - - - for (i = 0; i < n; i++) - { - u32 node_index; - - node_index = b->pre_data[i + 1]; + (*vlib_buffer_trace_trajectory_init_cb) (b); + } +} - if (node_index > vec_len (vnm->nodes)) - { - fformat (stderr, "Skip bogus node index %d\n", node_index); - continue; - } +#endif - fformat (stderr, "%v (%d)\n", vnm->nodes[node_index]->name, - node_index); - } - } - else +static inline void +add_trajectory_trace (vlib_buffer_t * b, u32 node_index) +{ +#if VLIB_BUFFER_TRACE_TRAJECTORY > 0 + if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0)) { - fformat (stderr, - "in vlib/buffers.h, #define VLIB_BUFFER_TRACE_TRAJECTORY 1\n"); + (*vlib_buffer_trace_trajectory_cb) (b, node_index); } +#endif } - static_always_inline u64 dispatch_node (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1002,15 +976,12 @@ dispatch_node (vlib_main_t * vm, if (VLIB_BUFFER_TRACE_TRAJECTORY && frame) { int i; - int log_index; u32 *from; from = vlib_frame_vector_args (frame); for (i = 0; i < frame->n_vectors; i++) { vlib_buffer_t *b = vlib_get_buffer (vm, from[i]); - ASSERT (b->pre_data[0] < 32); - log_index = b->pre_data[0]++ + 1; - b->pre_data[log_index] = node->node_index; + add_trajectory_trace (b, node->node_index); } n = node->function (vm, node, frame); } @@ -1135,13 +1106,13 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, { /* No next frame: so use dummy on stack. */ nf = &nf_dummy; - nf->flags = f->flags & VLIB_NODE_FLAG_TRACE; + nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE; nf->frame_index = ~p->frame_index; } else nf = vec_elt_at_index (nm->next_frames, p->next_frame_index); - ASSERT (f->flags & VLIB_FRAME_IS_ALLOCATED); + ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); /* Force allocation of new frame while current frame is being dispatched. */ @@ -1155,7 +1126,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, } /* Frame must be pending. */ - ASSERT (f->flags & VLIB_FRAME_PENDING); + ASSERT (f->frame_flags & VLIB_FRAME_PENDING); ASSERT (f->n_vectors > 0); /* Copy trace flag from next frame to node. @@ -1170,7 +1141,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, VLIB_NODE_STATE_POLLING, f, last_time_stamp); - f->flags &= ~VLIB_FRAME_PENDING; + f->frame_flags &= ~VLIB_FRAME_PENDING; /* Frame is ready to be used again, so restore it. */ if (restore_frame_index != ~0) @@ -1185,7 +1156,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, * longer part of the queue for that node and hence it cannot be * it's overspill. */ - ASSERT (!(f->flags & VLIB_FRAME_FREE_AFTER_DISPATCH)); + ASSERT (!(f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)); /* * NB: dispatching node n can result in the creation and scheduling @@ -1217,7 +1188,7 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, } else { - if (f->flags & VLIB_FRAME_FREE_AFTER_DISPATCH) + if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH) { ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH)); vlib_frame_free (vm, n, f); @@ -1306,6 +1277,7 @@ dispatch_process (vlib_main_t * vm, vlib_node_main_t *nm = &vm->node_main; vlib_node_runtime_t *node_runtime = &p->node_runtime; vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index); + u32 old_process_index; u64 t; uword n_vectors, is_suspend; @@ -1321,11 +1293,12 @@ dispatch_process (vlib_main_t * vm, f ? f->n_vectors : 0, /* is_after */ 0); /* Save away current process for suspend. */ + old_process_index = nm->current_process_index; nm->current_process_index = node->runtime_index; n_vectors = vlib_process_startup (vm, p, f); - nm->current_process_index = ~0; + nm->current_process_index = old_process_index; ASSERT (n_vectors != VLIB_PROCESS_RETURN_LONGJMP_RETURN); is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND; @@ -1453,6 +1426,13 @@ dispatch_suspended_process (vlib_main_t * vm, return t; } +void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak)); +void +vl_api_send_pending_rpc_requests (vlib_main_t * vm) +{ +} + + static_always_inline void vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { @@ -1505,6 +1485,9 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { vlib_node_runtime_t *n; + if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0)) + vl_api_send_pending_rpc_requests (vm); + if (!is_main) { vlib_worker_thread_barrier_check (); @@ -1513,13 +1496,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } /* Process pre-input nodes. */ - if (is_main) - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) - cpu_time_now = dispatch_node (vm, n, - VLIB_NODE_TYPE_PRE_INPUT, - VLIB_NODE_STATE_POLLING, - /* frame */ 0, - cpu_time_now); + vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) + cpu_time_now = dispatch_node (vm, n, + VLIB_NODE_TYPE_PRE_INPUT, + VLIB_NODE_STATE_POLLING, + /* frame */ 0, + cpu_time_now); /* Next process input nodes. */ vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) @@ -1534,13 +1516,19 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) /* Next handle interrupts. */ { + /* unlocked read, for performance */ uword l = _vec_len (nm->pending_interrupt_node_runtime_indices); uword i; - if (l > 0) + if (PREDICT_FALSE (l > 0)) { u32 *tmp; if (!is_main) - clib_spinlock_lock (&nm->pending_interrupt_lock); + { + clib_spinlock_lock (&nm->pending_interrupt_lock); + /* Re-read w/ lock held, in case another thread added an item */ + l = _vec_len (nm->pending_interrupt_node_runtime_indices); + } + tmp = nm->pending_interrupt_node_runtime_indices; nm->pending_interrupt_node_runtime_indices = last_node_runtime_indices; @@ -1560,6 +1548,13 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } } } + /* Input nodes may have added work to the pending vector. + Process pending vector until there is nothing left. + All pending vectors will be processed from input -> output. */ + for (i = 0; i < _vec_len (nm->pending_frames); i++) + cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now); + /* Reset pending vector for next iteration. */ + _vec_len (nm->pending_frames) = 0; if (is_main) { @@ -1578,7 +1573,6 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { uword i; - processes_timing_wheel_data: for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); i++) { @@ -1621,19 +1615,6 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) _vec_len (nm->data_from_advancing_timing_wheel) = 0; } } - - /* Input nodes may have added work to the pending vector. - Process pending vector until there is nothing left. - All pending vectors will be processed from input -> output. */ - for (i = 0; i < _vec_len (nm->pending_frames); i++) - cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now); - /* Reset pending vector for next iteration. */ - _vec_len (nm->pending_frames) = 0; - - /* Pending internal nodes may resume processes. */ - if (is_main && _vec_len (nm->data_from_advancing_timing_wheel) > 0) - goto processes_timing_wheel_data; - vlib_increment_main_loop_counter (vm); /* Record time stamp in case there are no enabled nodes and above @@ -1691,6 +1672,18 @@ dummy_queue_signal_callback (vlib_main_t * vm) { } +#define foreach_weak_reference_stub \ +_(vlib_map_stat_segment_init) \ +_(vpe_api_init) \ +_(vlibmemory_init) \ +_(map_api_segment_init) + +#define _(name) \ +clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \ +clib_error_t *name (vlib_main_t *vm) { return 0; } +foreach_weak_reference_stub; +#undef _ + /* Main function. */ int vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) @@ -1712,22 +1705,16 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) if (!vm->name) vm->name = "VLIB"; - vec_validate (vm->buffer_main, 0); - if (vlib_buffer_callbacks) + if ((error = unix_physmem_init (vm))) { - /* external plugin has registered own buffer callbacks - so we just copy them */ - vlib_buffer_main_t *bm = vm->buffer_main; - clib_memcpy (&bm->cb, vlib_buffer_callbacks, - sizeof (vlib_buffer_callbacks_t)); - bm->callbacks_registered = 1; + clib_error_report (error); + goto done; } - else + + if ((error = vlib_buffer_main_init (vm))) { - vlib_physmem_main_t *vpm = &vm->physmem_main; - vlib_buffer_cb_init (vm); - unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ ); - vlib_buffer_add_mem_range (vm, vpm->virtual.start, vpm->virtual.size); + clib_error_report (error); + goto done; } if ((error = vlib_thread_init (vm))) @@ -1736,6 +1723,12 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) goto done; } + if ((error = vlib_map_stat_segment_init (vm))) + { + clib_error_report (error); + goto done; + } + /* Register static nodes so that init functions may use them. */ vlib_register_all_static_nodes (vm); @@ -1755,6 +1748,25 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) goto done; } + /* Direct call / weak reference, for vlib standalone use-cases */ + if ((error = vpe_api_init (vm))) + { + clib_error_report (error); + goto done; + } + + if ((error = vlibmemory_init (vm))) + { + clib_error_report (error); + goto done; + } + + if ((error = map_api_segment_init (vm))) + { + clib_error_report (error); + goto done; + } + /* See unix/main.c; most likely already set up */ if (vm->init_functions_called == 0) vm->init_functions_called = hash_create (0, /* value bytes */ 0); @@ -1762,9 +1774,8 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) goto done; /* Create default buffer free list. */ - vlib_buffer_get_or_create_free_list (vm, - VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, - "default"); + vlib_buffer_create_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, + "default"); nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), CLIB_CACHE_LINE_BYTES); @@ -1778,6 +1789,9 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) 10e-6 /* timer period 10us */ , ~0 /* max expirations per call */ ); + vec_validate (vm->pending_rpc_requests, 0); + _vec_len (vm->pending_rpc_requests) = 0; + switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE)) { case VLIB_MAIN_LOOP_EXIT_NONE: