X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fthreads.c;h=03fda388edfa9a2628b657a8d673c29201c819b6;hb=e81f27ffb2a698737eae607b111d0611d221222f;hp=b470976d3d35a7f7b45410ca757a5e43c1feadcb;hpb=66c858385f3dc1c36682771424a8a9e5a6cd8355;p=vpp.git diff --git a/src/vlib/threads.c b/src/vlib/threads.c index b470976d3d3..03fda388edf 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -175,7 +175,6 @@ vlib_thread_init (vlib_main_t * vm) vlib_thread_main_t *tm = &vlib_thread_main; vlib_worker_thread_t *w; vlib_thread_registration_t *tr; - cpu_set_t cpuset; u32 n_vlib_mains = 1; u32 first_index = 1; u32 i; @@ -205,37 +204,35 @@ vlib_thread_init (vlib_main_t * vm) } /* grab cpu for main thread */ - if (tm->main_lcore == ~0) - { - /* if main-lcore is not set, we try to use lcore 1 */ - if (clib_bitmap_get (avail_cpu, 1)) - tm->main_lcore = 1; - else - tm->main_lcore = clib_bitmap_first_set (avail_cpu); - if (tm->main_lcore == (u8) ~ 0) - return clib_error_return (0, "no available cpus to be used for the" - " main thread"); - } - else + if (tm->main_lcore != ~0) { if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0) return clib_error_return (0, "cpu %u is not available to be used" " for the main thread", tm->main_lcore); + avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); } - avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); /* assume that there is socket 0 only if there is no data from sysfs */ if (!tm->cpu_socket_bitmap) tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); /* pin main thread to main_lcore */ - CPU_ZERO (&cpuset); - CPU_SET (tm->main_lcore, &cpuset); - pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); + if (tm->main_lcore != ~0) + { + cpu_set_t cpuset; + CPU_ZERO (&cpuset); + CPU_SET (tm->main_lcore, &cpuset); + if (pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), + &cpuset)) + { + return clib_error_return (0, "could not pin main thread to cpu %u", + tm->main_lcore); + } + } /* Set up thread 0 */ vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES); - _vec_len (vlib_worker_threads) = 1; + vec_set_len (vlib_worker_threads, 1); w = vlib_worker_threads; w->thread_mheap = clib_mem_get_heap (); w->thread_stack = vlib_thread_stacks[0]; @@ -312,7 +309,8 @@ vlib_thread_init (vlib_main_t * vm) if (c == ~0) return clib_error_return (0, "no available cpus to be used for" - " the '%s' thread", tr->name); + " the '%s' thread #%u", + tr->name, tr->count); avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0); avail_cpu = clib_bitmap_set (avail_cpu, c, 0); @@ -558,7 +556,7 @@ start_workers (vlib_main_t * vm) vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); - _vec_len (vgm->vlib_mains) = 0; + vec_set_len (vgm->vlib_mains, 0); vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES); if (n_vlib_mains > 1) @@ -638,13 +636,9 @@ start_workers (vlib_main_t * vm) sizeof (*vm_clone)); vm_clone->thread_index = worker_thread_index; - vm_clone->heap_base = w->thread_mheap; - vm_clone->heap_aligned_base = - (void *) (((uword) w->thread_mheap) & - ~(CLIB_CACHE_LINE_BYTES - 1)); vm_clone->pending_rpc_requests = 0; vec_validate (vm_clone->pending_rpc_requests, 0); - _vec_len (vm_clone->pending_rpc_requests) = 0; + vec_set_len (vm_clone->pending_rpc_requests, 0); clib_memset (&vm_clone->random_buffer, 0, sizeof (vm_clone->random_buffer)); clib_spinlock_init @@ -674,7 +668,7 @@ start_workers (vlib_main_t * vm) /* fork the frame dispatch queue */ nm_clone->pending_frames = 0; vec_validate (nm_clone->pending_frames, 10); - _vec_len (nm_clone->pending_frames) = 0; + vec_set_len (nm_clone->pending_frames, 0); /* fork nodes */ nm_clone->nodes = 0; @@ -711,8 +705,11 @@ start_workers (vlib_main_t * vm) vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], CLIB_CACHE_LINE_BYTES); clib_interrupt_init ( - &nm_clone->interrupts, + &nm_clone->input_node_interrupts, vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); + clib_interrupt_init ( + &nm_clone->pre_input_node_interrupts, + vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); @@ -810,28 +807,37 @@ start_workers (vlib_main_t * vm) { for (j = 0; j < tr->count; j++) { + w = vlib_worker_threads + worker_thread_index++; err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, w, 0); if (err) - clib_error_report (err); + clib_unix_error ("%U, thread %s init on cpu %d failed", + format_clib_error, err, tr->name, 0); } } else { uword c; - /* *INDENT-OFF* */ clib_bitmap_foreach (c, tr->coremask) { w = vlib_worker_threads + worker_thread_index++; err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, w, c); if (err) - clib_error_report (err); - } - /* *INDENT-ON* */ + clib_unix_error ("%U, thread %s init on cpu %d failed", + format_clib_error, err, tr->name, c); + } } } vlib_worker_thread_barrier_sync (vm); + { + clib_error_t *err; + err = vlib_call_init_exit_functions ( + vm, &vgm->num_workers_change_function_registrations, 1 /* call_once */, + 1 /* is_global */); + if (err) + clib_error_report (err); + } vlib_worker_thread_barrier_release (vm); return 0; } @@ -917,6 +923,17 @@ vlib_worker_thread_node_refork (void) vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES); vm_clone->error_main.counters_last_clear = old_counters_all_clear; + for (j = 0; j < vec_len (nm_clone->next_frames); j++) + { + vlib_next_frame_t *nf = &nm_clone->next_frames[j]; + if ((nf->flags & VLIB_FRAME_IS_ALLOCATED) && nf->frame != NULL) + { + vlib_frame_t *f = nf->frame; + nf->frame = NULL; + vlib_frame_free (vm_clone, f); + } + } + vec_free (nm_clone->next_frames); nm_clone->next_frames = vec_dup_aligned (nm->next_frames, CLIB_CACHE_LINE_BYTES); @@ -1015,8 +1032,11 @@ vlib_worker_thread_node_refork (void) vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], CLIB_CACHE_LINE_BYTES); clib_interrupt_resize ( - &nm_clone->interrupts, + &nm_clone->input_node_interrupts, vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); + clib_interrupt_resize ( + &nm_clone->pre_input_node_interrupts, + vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { @@ -1480,6 +1500,57 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) vm->clib_time.last_cpu_time, 1 /* leave */ ); } +static void +vlib_worker_sync_rpc (void *args) +{ + ASSERT (vlib_thread_is_main_w_barrier ()); + vlib_worker_threads->wait_before_barrier = 0; +} + +void +vlib_workers_sync (void) +{ + if (PREDICT_FALSE (!vlib_num_workers ())) + return; + + if (!(*vlib_worker_threads->wait_at_barrier) && + !clib_atomic_swap_rel_n (&vlib_worker_threads->wait_before_barrier, 1)) + { + u32 thread_index = vlib_get_thread_index (); + vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index, + sizeof (thread_index)); + vlib_worker_flush_pending_rpc_requests (vlib_get_main ()); + } + + /* Wait until main thread asks for barrier */ + while (!(*vlib_worker_threads->wait_at_barrier)) + ; + + /* Stop before barrier and make sure all threads are either + * at worker barrier or the barrier before it */ + clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, 1); + while (vlib_num_workers () > (*vlib_worker_threads->workers_at_barrier + + vlib_worker_threads->workers_before_barrier)) + ; +} + +void +vlib_workers_continue (void) +{ + if (PREDICT_FALSE (!vlib_num_workers ())) + return; + + clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, 1); + + /* Wait until all workers are done with work before barrier */ + while (vlib_worker_threads->done_work_before_barrier < + vlib_worker_threads->workers_before_barrier) + ; + + clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, -1); + clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, -1); +} + /** * Wait until each of the workers has been once around the track */ @@ -1516,6 +1587,19 @@ vlib_worker_wait_one_loop (void) return; } +void +vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm) +{ + vlib_main_t *vm_global = vlib_get_first_main (); + + ASSERT (vm != vm_global); + + clib_spinlock_lock_if_init (&vm_global->pending_rpc_lock); + vec_append (vm_global->pending_rpc_requests, vm->pending_rpc_requests); + vec_reset_length (vm->pending_rpc_requests); + clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock); +} + void vlib_worker_thread_fn (void *arg) { @@ -1549,12 +1633,18 @@ VLIB_REGISTER_THREAD (worker_thread_reg, static) = { }; /* *INDENT-ON* */ +extern clib_march_fn_registration + *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations; +extern clib_march_fn_registration + *vlib_frame_queue_dequeue_fn_march_fn_registrations; u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) { vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_main_t *vm = vlib_get_main (); vlib_frame_queue_main_t *fqm; vlib_frame_queue_t *fq; + vlib_node_t *node; int i; u32 num_threads; @@ -1566,11 +1656,24 @@ vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) vec_add2 (tm->frame_queue_mains, fqm, 1); + node = vlib_get_node (vm, fqm->node_index); + ASSERT (node); + if (node->aux_offset) + { + fqm->frame_queue_dequeue_fn = + CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_with_aux_fn); + } + else + { + fqm->frame_queue_dequeue_fn = + CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_fn); + } + fqm->node_index = node_index; fqm->frame_queue_nelts = frame_queue_nelts; vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1); - _vec_len (fqm->vlib_frame_queues) = 0; + vec_set_len (fqm->vlib_frame_queues, 0); for (i = 0; i < tm->n_vlib_mains; i++) { fq = vlib_frame_queue_alloc (frame_queue_nelts); @@ -1606,12 +1709,17 @@ vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size) clib_error_t * threads_init (vlib_main_t * vm) { + const vlib_thread_main_t *tm = vlib_get_thread_main (); + + if (tm->main_lcore == ~0 && tm->n_vlib_mains > 1) + return clib_error_return (0, "Configuration error, a main core must " + "be specified when using worker threads"); + return 0; } VLIB_INIT_FUNCTION (threads_init); - static clib_error_t * show_clock_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd)