#include <vppinfra/time_range.h>
#include <vppinfra/interrupt.h>
#include <vppinfra/linux/sysfs.h>
+#include <vppinfra/bitmap.h>
+#include <vppinfra/unix.h>
#include <vlib/vlib.h>
#include <vlib/threads.h>
vlib_thread_main_t *tm = &vlib_thread_main;
vlib_worker_thread_t *w;
vlib_thread_registration_t *tr;
- cpu_set_t cpuset;
u32 n_vlib_mains = 1;
u32 first_index = 1;
u32 i;
ASSERT (stats_num_worker_threads_dir_index != ~0);
/* get bitmaps of active cpu cores and sockets */
- tm->cpu_core_bitmap =
- clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
- tm->cpu_socket_bitmap =
- clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
+ tm->cpu_core_bitmap = os_get_online_cpu_core_bitmap ();
+ tm->cpu_socket_bitmap = os_get_online_cpu_node_bitmap ();
avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
}
/* grab cpu for main thread */
- if (tm->main_lcore == ~0)
- {
- /* if main-lcore is not set, we try to use lcore 1 */
- if (clib_bitmap_get (avail_cpu, 1))
- tm->main_lcore = 1;
- else
- tm->main_lcore = clib_bitmap_first_set (avail_cpu);
- if (tm->main_lcore == (u8) ~ 0)
- return clib_error_return (0, "no available cpus to be used for the"
- " main thread");
- }
- else
+ if (tm->main_lcore != ~0)
{
if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
return clib_error_return (0, "cpu %u is not available to be used"
" for the main thread", tm->main_lcore);
+ avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
}
- avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
/* assume that there is socket 0 only if there is no data from sysfs */
if (!tm->cpu_socket_bitmap)
tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
/* pin main thread to main_lcore */
- CPU_ZERO (&cpuset);
- CPU_SET (tm->main_lcore, &cpuset);
- pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
+ if (tm->main_lcore != ~0)
+ {
+ cpu_set_t cpuset;
+ CPU_ZERO (&cpuset);
+ CPU_SET (tm->main_lcore, &cpuset);
+ if (pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t),
+ &cpuset))
+ {
+ return clib_error_return (0, "could not pin main thread to cpu %u",
+ tm->main_lcore);
+ }
+ }
/* Set up thread 0 */
vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
- _vec_len (vlib_worker_threads) = 1;
+ vec_set_len (vlib_worker_threads, 1);
w = vlib_worker_threads;
w->thread_mheap = clib_mem_get_heap ();
w->thread_stack = vlib_thread_stacks[0];
if (c == ~0)
return clib_error_return (0,
"no available cpus to be used for"
- " the '%s' thread", tr->name);
+ " the '%s' thread #%u",
+ tr->name, tr->count);
avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
clib_sysfs_read ((char *) p, "%d", &core_id);
vec_reset_length (p);
- /* *INDENT-OFF* */
- clib_sysfs_read ("/sys/devices/system/node/online", "%U",
- unformat_bitmap_list, &nbmp);
+ nbmp = os_get_online_cpu_node_bitmap ();
clib_bitmap_foreach (node, nbmp) {
p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0);
clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp);
vec_reset_length (cbmp);
vec_reset_length (p);
}
- /* *INDENT-ON* */
+
vec_free (nbmp);
vec_free (cbmp);
vec_free (p);
vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1,
CLIB_CACHE_LINE_BYTES);
- _vec_len (vgm->vlib_mains) = 0;
+ vec_set_len (vgm->vlib_mains, 0);
vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
if (n_vlib_mains > 1)
sizeof (*vm_clone));
vm_clone->thread_index = worker_thread_index;
- vm_clone->heap_base = w->thread_mheap;
- vm_clone->heap_aligned_base =
- (void *) (((uword) w->thread_mheap) &
- ~(CLIB_CACHE_LINE_BYTES - 1));
vm_clone->pending_rpc_requests = 0;
vec_validate (vm_clone->pending_rpc_requests, 0);
- _vec_len (vm_clone->pending_rpc_requests) = 0;
+ vec_set_len (vm_clone->pending_rpc_requests, 0);
clib_memset (&vm_clone->random_buffer, 0,
sizeof (vm_clone->random_buffer));
clib_spinlock_init
/* fork the frame dispatch queue */
nm_clone->pending_frames = 0;
vec_validate (nm_clone->pending_frames, 10);
- _vec_len (nm_clone->pending_frames) = 0;
+ vec_set_len (nm_clone->pending_frames, 0);
/* fork nodes */
nm_clone->nodes = 0;
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_init (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_init (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
vlib_node_t *n = vlib_get_node (vm, rt->node_index);
{
for (j = 0; j < tr->count; j++)
{
+
w = vlib_worker_threads + worker_thread_index++;
err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
w, 0);
if (err)
- clib_error_report (err);
+ clib_unix_error ("%U, thread %s init on cpu %d failed",
+ format_clib_error, err, tr->name, 0);
}
}
else
{
uword c;
- /* *INDENT-OFF* */
clib_bitmap_foreach (c, tr->coremask) {
w = vlib_worker_threads + worker_thread_index++;
err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
w, c);
if (err)
- clib_error_report (err);
- }
- /* *INDENT-ON* */
+ clib_unix_error ("%U, thread %s init on cpu %d failed",
+ format_clib_error, err, tr->name, c);
+ }
}
}
vlib_worker_thread_barrier_sync (vm);
+ {
+ clib_error_t *err;
+ err = vlib_call_init_exit_functions (
+ vm, &vgm->num_workers_change_function_registrations, 1 /* call_once */,
+ 1 /* is_global */);
+ if (err)
+ clib_error_report (err);
+ }
vlib_worker_thread_barrier_release (vm);
return 0;
}
vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
vm_clone->error_main.counters_last_clear = old_counters_all_clear;
+ for (j = 0; j < vec_len (nm_clone->next_frames); j++)
+ {
+ vlib_next_frame_t *nf = &nm_clone->next_frames[j];
+ if ((nf->flags & VLIB_FRAME_IS_ALLOCATED) && nf->frame != NULL)
+ {
+ vlib_frame_t *f = nf->frame;
+ nf->frame = NULL;
+ vlib_frame_free (vm_clone, f);
+ }
+ }
+
vec_free (nm_clone->next_frames);
nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
CLIB_CACHE_LINE_BYTES);
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_resize (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_resize (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
u32 thread_index = vlib_get_thread_index ();
vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index,
sizeof (thread_index));
+ vlib_worker_flush_pending_rpc_requests (vlib_get_main ());
}
/* Wait until main thread asks for barrier */
return;
}
+void
+vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm)
+{
+ vlib_main_t *vm_global = vlib_get_first_main ();
+
+ ASSERT (vm != vm_global);
+
+ clib_spinlock_lock_if_init (&vm_global->pending_rpc_lock);
+ vec_append (vm_global->pending_rpc_requests, vm->pending_rpc_requests);
+ vec_reset_length (vm->pending_rpc_requests);
+ clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock);
+}
+
void
vlib_worker_thread_fn (void *arg)
{
};
/* *INDENT-ON* */
+extern clib_march_fn_registration
+ *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations;
+extern clib_march_fn_registration
+ *vlib_frame_queue_dequeue_fn_march_fn_registrations;
u32
vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
{
vlib_thread_main_t *tm = vlib_get_thread_main ();
+ vlib_main_t *vm = vlib_get_main ();
vlib_frame_queue_main_t *fqm;
vlib_frame_queue_t *fq;
+ vlib_node_t *node;
int i;
u32 num_threads;
vec_add2 (tm->frame_queue_mains, fqm, 1);
+ node = vlib_get_node (vm, fqm->node_index);
+ ASSERT (node);
+ if (node->aux_offset)
+ {
+ fqm->frame_queue_dequeue_fn =
+ CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_with_aux_fn);
+ }
+ else
+ {
+ fqm->frame_queue_dequeue_fn =
+ CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_fn);
+ }
+
fqm->node_index = node_index;
fqm->frame_queue_nelts = frame_queue_nelts;
vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1);
- _vec_len (fqm->vlib_frame_queues) = 0;
+ vec_set_len (fqm->vlib_frame_queues, 0);
for (i = 0; i < tm->n_vlib_mains; i++)
{
fq = vlib_frame_queue_alloc (frame_queue_nelts);
clib_error_t *
threads_init (vlib_main_t * vm)
{
+ const vlib_thread_main_t *tm = vlib_get_thread_main ();
+
+ if (tm->main_lcore == ~0 && tm->n_vlib_mains > 1)
+ return clib_error_return (0, "Configuration error, a main core must "
+ "be specified when using worker threads");
+
return 0;
}
VLIB_INIT_FUNCTION (threads_init);
-
static clib_error_t *
show_clock_command_fn (vlib_main_t * vm,
unformat_input_t * input, vlib_cli_command_t * cmd)
Code Review / vpp.git / blobdiff