X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fthreads.c;h=7efddff54e802a33854d704f60a1a608c775974c;hb=f0ca1e8d92114582ec9142bd15a40f1eb0102793;hp=e9d9cb5a158d9ea5267d93c324b41e399c55776e;hpb=197180031bad1e51ee032d30d8a095a51207454c;p=vpp.git diff --git a/src/vlib/threads.c b/src/vlib/threads.c index e9d9cb5a158..7efddff54e8 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -22,13 +22,9 @@ #include #include -#include #include -DECLARE_CJ_GLOBAL_LOG; - - u32 vl (void *p) { @@ -256,21 +252,6 @@ vlib_thread_init (vlib_main_t * vm) } avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); - /* - * Determine if the number of workers is greater than 0. - * If so, mark CPU 0 unavailable so workers will be numbered after main. - */ - u32 n_workers = 0; - uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - if (p != 0) - { - vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; - int worker_thread_count = tr->count; - n_workers = worker_thread_count; - } - if (tm->skip_cores == 0 && n_workers) - avail_cpu = clib_bitmap_set (avail_cpu, 0, 0); - /* assume that there is socket 0 only if there is no data from sysfs */ if (!tm->cpu_socket_bitmap) tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); @@ -339,25 +320,37 @@ vlib_thread_init (vlib_main_t * vm) { uword c; /* *INDENT-OFF* */ - clib_bitmap_foreach (c, tr->coremask, ({ + clib_bitmap_foreach (c, tr->coremask) { if (clib_bitmap_get(avail_cpu, c) == 0) return clib_error_return (0, "cpu %u is not available to be used" " for the '%s' thread",c, tr->name); avail_cpu = clib_bitmap_set(avail_cpu, c, 0); - })); + } /* *INDENT-ON* */ } else { for (j = 0; j < tr->count; j++) { + /* Do not use CPU 0 by default - leave it to the host and IRQs */ + uword avail_c0 = clib_bitmap_get (avail_cpu, 0); + avail_cpu = clib_bitmap_set (avail_cpu, 0, 0); + uword c = clib_bitmap_first_set (avail_cpu); + /* Use CPU 0 as a last resort */ + if (c == ~0 && avail_c0) + { + c = 0; + avail_c0 = 0; + } + if (c == ~0) return clib_error_return (0, "no available cpus to be used for" " the '%s' thread", tr->name); + avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0); avail_cpu = clib_bitmap_set (avail_cpu, c, 0); tr->coremask = clib_bitmap_set (tr->coremask, c, 1); } @@ -588,6 +581,7 @@ vlib_worker_thread_bootstrap_fn (void *arg) __os_thread_index = w - vlib_worker_threads; + vlib_process_start_switch_stack (vlib_mains[__os_thread_index], 0); rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, (uword) arg, w->thread_stack + VLIB_THREAD_STACK_SIZE); @@ -612,14 +606,14 @@ vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id) /* *INDENT-OFF* */ clib_sysfs_read ("/sys/devices/system/node/online", "%U", unformat_bitmap_list, &nbmp); - clib_bitmap_foreach (node, nbmp, ({ + clib_bitmap_foreach (node, nbmp) { p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0); clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp); if (clib_bitmap_get (cbmp, cpu_id)) numa_id = node; vec_reset_length (cbmp); vec_reset_length (p); - })); + } /* *INDENT-ON* */ vec_free (nbmp); vec_free (cbmp); @@ -632,6 +626,7 @@ vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id) static clib_error_t * vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) { + clib_mem_main_t *mm = &clib_mem_main; vlib_thread_main_t *tm = &vlib_thread_main; void *(*fp_arg) (void *) = fp; void *numa_heap; @@ -640,19 +635,22 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) vlib_get_thread_core_numa (w, cpu_id); /* Set up NUMA-bound heap if indicated */ - if (clib_per_numa_mheaps[w->numa_id] == 0) + if (mm->per_numa_mheaps[w->numa_id] == 0) { /* If the user requested a NUMA heap, create it... */ if (tm->numa_heap_size) { - numa_heap = clib_mem_init_thread_safe_numa - (0 /* DIY */ , tm->numa_heap_size, w->numa_id); - clib_per_numa_mheaps[w->numa_id] = numa_heap; + clib_mem_set_numa_affinity (w->numa_id, 1 /* force */ ); + numa_heap = clib_mem_create_heap (0 /* DIY */ , tm->numa_heap_size, + 1 /* is_locked */ , + "numa %u heap", w->numa_id); + clib_mem_set_default_numa_affinity (); + mm->per_numa_mheaps[w->numa_id] = numa_heap; } else { /* Or, use the main heap */ - clib_per_numa_mheaps[w->numa_id] = w->thread_mheap; + mm->per_numa_mheaps[w->numa_id] = w->thread_mheap; } } @@ -687,7 +685,7 @@ start_workers (vlib_main_t * vm) vlib_node_runtime_t *rt; u32 n_vlib_mains = tm->n_vlib_mains; u32 worker_thread_index; - u8 *main_heap = clib_mem_get_per_cpu_heap (); + clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap (); vec_reset_length (vlib_worker_threads); @@ -706,16 +704,18 @@ start_workers (vlib_main_t * vm) clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); vm->elog_main.lock[0] = 0; - if (n_vlib_mains > 1) - { - /* Replace hand-crafted length-1 vector with a real vector */ - vlib_mains = 0; + clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks, + &vm->worker_thread_main_loop_callback_lock); + + /* Replace hand-crafted length-1 vector with a real vector */ + vlib_mains = 0; - vec_validate_aligned (vlib_mains, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - _vec_len (vlib_mains) = 0; - vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (vlib_mains, n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); + _vec_len (vlib_mains) = 0; + vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES); + if (n_vlib_mains > 1) + { vlib_worker_threads->wait_at_barrier = clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); vlib_worker_threads->workers_at_barrier = @@ -740,6 +740,7 @@ start_workers (vlib_main_t * vm) vm->barrier_no_close_before = 0; worker_thread_index = 1; + clib_spinlock_init (&vm->worker_thread_main_loop_callback_lock); for (i = 0; i < vec_len (tm->registrations); i++) { @@ -758,8 +759,10 @@ start_workers (vlib_main_t * vm) vec_add2 (vlib_worker_threads, w, 1); /* Currently unused, may not really work */ if (tr->mheap_size) - w->thread_mheap = create_mspace (tr->mheap_size, - 0 /* unlocked */ ); + w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size, + /* unlocked */ 0, + "%s%d heap", + tr->name, k); else w->thread_mheap = main_heap; @@ -796,6 +799,11 @@ start_workers (vlib_main_t * vm) _vec_len (vm_clone->pending_rpc_requests) = 0; clib_memset (&vm_clone->random_buffer, 0, sizeof (vm_clone->random_buffer)); + clib_spinlock_init + (&vm_clone->worker_thread_main_loop_callback_lock); + clib_callback_data_init + (&vm_clone->vlib_node_runtime_perf_callbacks, + &vm_clone->worker_thread_main_loop_callback_lock); nm = &vlib_mains[0]->node_main; nm_clone = &vm_clone->node_main; @@ -923,8 +931,10 @@ start_workers (vlib_main_t * vm) vec_add2 (vlib_worker_threads, w, 1); if (tr->mheap_size) { - w->thread_mheap = - create_mspace (tr->mheap_size, 0 /* locked */ ); + w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size, + /* locked */ 0, + "%s%d heap", + tr->name, j); } else w->thread_mheap = main_heap; @@ -966,13 +976,13 @@ start_workers (vlib_main_t * vm) { uword c; /* *INDENT-OFF* */ - clib_bitmap_foreach (c, tr->coremask, ({ + clib_bitmap_foreach (c, tr->coremask) { w = vlib_worker_threads + worker_thread_index++; err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, w, c); if (err) clib_error_report (err); - })); + } /* *INDENT-ON* */ } } @@ -1299,6 +1309,10 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) return clib_error_return (0, "corelist cannot be set for '%s' threads", name); + if (tr->count) + return clib_error_return + (0, "core placement of '%s' threads is already configured", + name); tr->coremask = bitmap; tr->count = clib_bitmap_count_set_bits (tr->coremask); @@ -1317,9 +1331,14 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) return clib_error_return (0, "no such thread type 3 '%s'", name); tr = (vlib_thread_registration_t *) p[0]; + if (tr->fixed_count) return clib_error_return - (0, "number of %s threads not configurable", tr->name); + (0, "number of '%s' threads not configurable", name); + if (tr->count) + return clib_error_return + (0, "number of '%s' threads is already configured", name); + tr->count = count; } else @@ -1432,6 +1451,18 @@ vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm) *vlib_worker_threads->wait_at_barrier = 0; } +/** + * Return true if the wroker thread barrier is held + */ +u8 +vlib_worker_thread_barrier_held (void) +{ + if (vec_len (vlib_mains) < 2) + return (1); + + return (*vlib_worker_threads->wait_at_barrier == 1); +} + void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name) { @@ -1463,6 +1494,10 @@ vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name) return; } + if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) + clib_call_callbacks (vm->barrier_perf_callbacks, vm, + vm->clib_time.last_cpu_time, 0 /* enter */ ); + /* * Need data to decide if we're working hard enough to honor * the barrier hold-down timer. @@ -1626,6 +1661,44 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) barrier_trace_release (t_entry, t_closed_total, t_update_main); + if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) + clib_call_callbacks (vm->barrier_perf_callbacks, vm, + vm->clib_time.last_cpu_time, 1 /* leave */ ); +} + +/** + * Wait until each of the workers has been once around the track + */ +void +vlib_worker_wait_one_loop (void) +{ + ASSERT (vlib_get_thread_index () == 0); + + if (vec_len (vlib_mains) < 2) + return; + + if (vlib_worker_thread_barrier_held ()) + return; + + u32 *counts = 0; + u32 ii; + + vec_validate (counts, vec_len (vlib_mains) - 1); + + /* record the current loop counts */ + vec_foreach_index (ii, vlib_mains) + counts[ii] = vlib_mains[ii]->main_loop_count; + + /* spin until each changes, apart from the main thread, or we'd be + * a while */ + for (ii = 1; ii < vec_len (counts); ii++) + { + while (counts[ii] == vlib_mains[ii]->main_loop_count) + CLIB_PAUSE (); + } + + vec_free (counts); + return; } /* @@ -1775,6 +1848,8 @@ vlib_worker_thread_fn (void *arg) vlib_main_t *vm = vlib_get_main (); clib_error_t *e; + vlib_process_finish_switch_stack (vm); + ASSERT (vm->thread_index == vlib_get_thread_index ()); vlib_worker_thread_init (w); @@ -1808,17 +1883,19 @@ vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) vlib_frame_queue_main_t *fqm; vlib_frame_queue_t *fq; int i; + u32 num_threads; if (frame_queue_nelts == 0) frame_queue_nelts = FRAME_QUEUE_MAX_NELTS; - ASSERT (frame_queue_nelts >= 8); + num_threads = 1 /* main thread */ + tm->n_threads; + ASSERT (frame_queue_nelts >= 8 + num_threads); vec_add2 (tm->frame_queue_mains, fqm, 1); fqm->node_index = node_index; fqm->frame_queue_nelts = frame_queue_nelts; - fqm->queue_hi_thresh = frame_queue_nelts - 2; + fqm->queue_hi_thresh = frame_queue_nelts - num_threads; vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1); vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1); @@ -1933,6 +2010,12 @@ VLIB_CLI_COMMAND (f_command, static) = }; /* *INDENT-ON* */ +vlib_thread_main_t * +vlib_get_thread_main_not_inline (void) +{ + return vlib_get_thread_main (); +} + /* * fd.io coding-style-patch-verification: ON *