From: Dave Barach Date: Mon, 3 Jun 2019 23:48:22 +0000 (-0400) Subject: sort worker-thread init functions in advance X-Git-Tag: v20.01-rc0~487 X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;ds=sidebyside;h=c602b384ac022f70690a3a7c711149f7cb63ad12;p=vpp.git sort worker-thread init functions in advance Otherwise, all N worker threads try to sort the list at the same time: a good way to have a bad day. This approach performs *far* better than maintaing order by adding a spin-lock. By direct measurement w/ elog + g2: 11 threads execute the per-thread init function list in 22us, vs. 50ms with a CLIB_PAUSE() enabled spin-lock. Change-Id: I1745f2a213c0561260139a60114dcb981e0c64e5 Signed-off-by: Dave Barach --- diff --git a/src/vlib/init.c b/src/vlib/init.c index 8010e9e97cd..135f9d582e9 100644 --- a/src/vlib/init.c +++ b/src/vlib/init.c @@ -72,7 +72,7 @@ comma_split (u8 * s, u8 ** a, u8 ** b) * @returns 0 on success, otherwise a clib_error_t *. */ -static clib_error_t *init_exit_function_sort +clib_error_t *vlib_sort_init_exit_functions (_vlib_init_function_list_elt_t ** head) { uword *index_by_name; @@ -329,15 +329,15 @@ again: * A and B - and it leads to hugely annoying debugging exercises. */ -clib_error_t * -vlib_call_init_exit_functions (vlib_main_t * vm, - _vlib_init_function_list_elt_t ** headp, - int call_once) +static inline clib_error_t * +call_init_exit_functions_internal (vlib_main_t * vm, + _vlib_init_function_list_elt_t ** headp, + int call_once, int do_sort) { clib_error_t *error = 0; _vlib_init_function_list_elt_t *i; - if ((error = init_exit_function_sort (headp))) + if (do_sort && (error = vlib_sort_init_exit_functions (headp))) return (error); i = *headp; @@ -356,6 +356,24 @@ vlib_call_init_exit_functions (vlib_main_t * vm, return error; } +clib_error_t * +vlib_call_init_exit_functions (vlib_main_t * vm, + _vlib_init_function_list_elt_t ** headp, + int call_once) +{ + return call_init_exit_functions_internal (vm, headp, call_once, + 1 /* do_sort */ ); +} + +clib_error_t * +vlib_call_init_exit_functions_no_sort (vlib_main_t * vm, + _vlib_init_function_list_elt_t ** + headp, int call_once) +{ + return call_init_exit_functions_internal (vm, headp, call_once, + 0 /* do_sort */ ); +} + clib_error_t * vlib_call_all_init_functions (vlib_main_t * vm) { diff --git a/src/vlib/init.h b/src/vlib/init.h index 6d2711489d8..fc638013efc 100644 --- a/src/vlib/init.h +++ b/src/vlib/init.h @@ -328,7 +328,11 @@ clib_error_t *vlib_call_all_main_loop_exit_functions (struct vlib_main_t *vm); clib_error_t *vlib_call_init_exit_functions (struct vlib_main_t *vm, _vlib_init_function_list_elt_t ** headp, int call_once); - +clib_error_t *vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm, + _vlib_init_function_list_elt_t + ** headp, int call_once); +clib_error_t *vlib_sort_init_exit_functions (_vlib_init_function_list_elt_t + **); #define foreach_vlib_module_reference \ _ (node_cli) \ _ (trace_cli) diff --git a/src/vlib/main.c b/src/vlib/main.c index b6006e8bab0..43400f8594c 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1721,6 +1721,14 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (is_main) { uword i; + + /* + * Perform an initial barrier sync. Pays no attention to + * the barrier sync hold-down timer scheme, which won't work + * at this point in time. + */ + vlib_worker_thread_initial_barrier_sync_and_release (vm); + nm->current_process_index = ~0; for (i = 0; i < vec_len (nm->processes); i++) cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0, @@ -2090,6 +2098,9 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ ))) goto done; + /* Sort per-thread init functions before we start threads */ + vlib_sort_init_exit_functions (&vm->worker_init_function_registrations); + /* Call all main loop enter functions. */ { clib_error_t *sub_error; diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 22fa5f12ecd..4e4d60a5e62 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1389,6 +1389,31 @@ vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which) #define BARRIER_MINIMUM_OPEN_FACTOR 3 #endif +void +vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm) +{ + f64 deadline; + f64 now = vlib_time_now (vm); + u32 count = vec_len (vlib_mains) - 1; + + /* No worker threads? */ + if (count == 0) + return; + + deadline = now + BARRIER_SYNC_TIMEOUT; + *vlib_worker_threads->wait_at_barrier = 1; + while (*vlib_worker_threads->workers_at_barrier != count) + { + if ((now = vlib_time_now (vm)) > deadline) + { + fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); + os_panic (); + } + CLIB_PAUSE (); + } + *vlib_worker_threads->wait_at_barrier = 0; +} + void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name) { @@ -1732,15 +1757,15 @@ vlib_worker_thread_fn (void *arg) clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); - /* Wait until the dpdk init sequence is complete */ - while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) - vlib_worker_thread_barrier_check (); - - e = vlib_call_init_exit_functions + e = vlib_call_init_exit_functions_no_sort (vm, &vm->worker_init_function_registrations, 1 /* call_once */ ); if (e) clib_error_report (e); + /* Wait until the dpdk init sequence is complete */ + while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) + vlib_worker_thread_barrier_check (); + vlib_worker_loop (vm); } diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 5a295e03319..fc34074127f 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -206,6 +206,7 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name); void vlib_worker_thread_barrier_release (vlib_main_t * vm); +void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm); void vlib_worker_thread_node_refork (void); static_always_inline uword