X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fthreads.h;h=9d9d387d1550ed3a92345a702f75a219ee26f4f0;hb=06111a837;hp=bb7c164c2e3e93cb2105e980e001a5638a4eb77a;hpb=c3a06556d1a4a63646d4cc7aa76274177a56c13f;p=vpp.git diff --git a/src/vlib/threads.h b/src/vlib/threads.h index bb7c164c2e3..9d9d387d155 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -16,10 +16,9 @@ #define included_vlib_threads_h #include +#include #include -extern vlib_main_t **vlib_mains; - void vlib_set_thread_name (char *name); /* arg is actually a vlib__thread_t * */ @@ -110,7 +109,7 @@ typedef struct long lwp; int cpu_id; int core_id; - int socket_id; + int numa_id; pthread_t thread_id; } vlib_worker_thread_t; @@ -180,9 +179,6 @@ int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type); -int -vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm); - void vlib_worker_thread_node_runtime_update (void); void vlib_create_worker_threads (vlib_main_t * vm, int n, @@ -201,11 +197,18 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); #define BARRIER_SYNC_TIMEOUT (1.0) #endif -#define vlib_worker_thread_barrier_sync(X) {vlib_worker_threads[0].barrier_caller=__FUNCTION__;vlib_worker_thread_barrier_sync_int(X);} +#define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);} -void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm); +void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, + const char *func_name); void vlib_worker_thread_barrier_release (vlib_main_t * vm); +u8 vlib_worker_thread_barrier_held (void); +void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm); void vlib_worker_thread_node_refork (void); +/** + * Wait until each of the workers has been once around the track + */ +void vlib_worker_wait_one_loop (void); static_always_inline uword vlib_get_thread_index (void) @@ -223,36 +226,24 @@ vlib_smp_unsafe_warning (void) } } -typedef enum +always_inline int +__foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p) { - VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0, - VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX, -} vlib_fork_fixup_t; - -void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which); - -#define foreach_vlib_main(body) \ -do { \ - vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ - int ii; \ - \ - for (ii = 0; ii < vec_len (vlib_mains); ii++) \ - { \ - this_vlib_main = vlib_mains[ii]; \ - ASSERT (ii == 0 || \ - this_vlib_main->parked_at_barrier == 1); \ - if (this_vlib_main) \ - vec_add1 (__vlib_mains, this_vlib_main); \ - } \ - \ - for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ - { \ - this_vlib_main = __vlib_mains[ii]; \ - /* body uses this_vlib_main... */ \ - (body); \ - } \ - vec_free (__vlib_mains); \ -} while (0); + vlib_main_t *vm; + u32 index = ii - (vlib_main_t *) 0; + + if (index >= vec_len (vlib_global_main.vlib_mains)) + return 0; + + *p = vm = vlib_global_main.vlib_mains[index]; + ASSERT (index == 0 || vm->parked_at_barrier == 1); + return 1; +} + +#define foreach_vlib_main() \ + for (vlib_main_t *ii = 0, *this_vlib_main; \ + __foreach_vlib_main_helper (ii, &this_vlib_main); ii++) \ + if (this_vlib_main) #define foreach_sched_policy \ _(SCHED_OTHER, OTHER, "other") \ @@ -336,6 +327,10 @@ typedef struct /* callbacks */ vlib_thread_callbacks_t cb; int extern_thread_mgmt; + + /* NUMA-bound heap size */ + uword numa_heap_size; + } vlib_thread_main_t; extern vlib_thread_main_t vlib_thread_main; @@ -390,10 +385,15 @@ vlib_worker_thread_barrier_check (void) { if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier)) { + vlib_global_main_t *vgm = vlib_get_global_main (); vlib_main_t *vm = vlib_get_main (); u32 thread_index = vm->thread_index; f64 t = vlib_time_now (vm); + if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) + clib_call_callbacks (vm->barrier_perf_callbacks, vm, + vm->clib_time.last_cpu_time, 0 /* enter */ ); + if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled)) { vlib_worker_thread_t *w = vlib_worker_threads + thread_index; @@ -409,22 +409,36 @@ vlib_worker_thread_barrier_check (void) u32 thread_index; } __clib_packed *ed; - ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, - w->elog_track); + ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track); ed->thread_index = thread_index; } - clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1); if (CLIB_DEBUG > 0) { vm = vlib_get_main (); vm->parked_at_barrier = 1; } + clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1); while (*vlib_worker_threads->wait_at_barrier) ; + + /* + * Recompute the offset from thread-0 time. + * Note that vlib_time_now adds vm->time_offset, so + * clear it first. Save the resulting idea of "now", to + * see how well we're doing. See show_clock_command_fn(...) + */ + { + f64 now; + vm->time_offset = 0.0; + now = vlib_time_now (vm); + vm->time_offset = vgm->vlib_mains[0]->time_last_barrier_release - now; + vm->time_last_barrier_release = vlib_time_now (vm); + } + if (CLIB_DEBUG > 0) vm->parked_at_barrier = 0; - clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1); + clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1); if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required)) { @@ -450,8 +464,8 @@ vlib_worker_thread_barrier_check (void) } vlib_worker_thread_node_refork (); - clib_smp_atomic_add (vlib_worker_threads->node_reforks_required, - -1); + clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required, + -1); while (*vlib_worker_threads->node_reforks_required) ; } @@ -472,11 +486,14 @@ vlib_worker_thread_barrier_check (void) u32 duration; } __clib_packed *ed; - ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, - w->elog_track); + ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track); ed->thread_index = thread_index; ed->duration = (int) (1000000.0 * t); } + + if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) + clib_call_callbacks (vm->barrier_perf_callbacks, vm, + vm->clib_time.last_cpu_time, 1 /* leave */ ); } } @@ -486,7 +503,7 @@ vlib_get_worker_vlib_main (u32 worker_index) vlib_main_t *vm; vlib_thread_main_t *tm = &vlib_thread_main; ASSERT (worker_index < tm->n_vlib_mains - 1); - vm = vlib_mains[worker_index + 1]; + vm = vlib_get_main_by_index (worker_index + 1); ASSERT (vm); return vm; } @@ -519,7 +536,7 @@ vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index) fq = fqm->vlib_frame_queues[index]; ASSERT (fq); - new_tail = __sync_add_and_fetch (&fq->tail, 1); + new_tail = clib_atomic_add_fetch (&fq->tail, 1); /* Wait until a ring slot is available */ while (new_tail >= fq->head_hint + fq->nelts) @@ -596,8 +613,9 @@ void vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t * args); void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size); +void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id); +vlib_thread_main_t *vlib_get_thread_main_not_inline (void); -u32 elog_global_id_for_msg_name (const char *msg_name); #endif /* included_vlib_threads_h */ /*