#include <vppinfra/time_range.h>
#include <vppinfra/interrupt.h>
#include <vppinfra/linux/sysfs.h>
+#include <vppinfra/bitmap.h>
+#include <vppinfra/unix.h>
#include <vlib/vlib.h>
#include <vlib/threads.h>
ASSERT (stats_num_worker_threads_dir_index != ~0);
/* get bitmaps of active cpu cores and sockets */
- tm->cpu_core_bitmap =
- clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
- tm->cpu_socket_bitmap =
- clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
+ tm->cpu_core_bitmap = os_get_online_cpu_core_bitmap ();
+ tm->cpu_socket_bitmap = os_get_online_cpu_node_bitmap ();
avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
cpu_set_t cpuset;
CPU_ZERO (&cpuset);
CPU_SET (tm->main_lcore, &cpuset);
- pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
+ if (pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t),
+ &cpuset))
+ {
+ return clib_error_return (0, "could not pin main thread to cpu %u",
+ tm->main_lcore);
+ }
}
/* Set up thread 0 */
if (c == ~0)
return clib_error_return (0,
"no available cpus to be used for"
- " the '%s' thread", tr->name);
+ " the '%s' thread #%u",
+ tr->name, tr->count);
avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
clib_sysfs_read ((char *) p, "%d", &core_id);
vec_reset_length (p);
- /* *INDENT-OFF* */
- clib_sysfs_read ("/sys/devices/system/node/online", "%U",
- unformat_bitmap_list, &nbmp);
+ nbmp = os_get_online_cpu_node_bitmap ();
clib_bitmap_foreach (node, nbmp) {
p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0);
clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp);
vec_reset_length (cbmp);
vec_reset_length (p);
}
- /* *INDENT-ON* */
+
vec_free (nbmp);
vec_free (cbmp);
vec_free (p);
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_init (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_init (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
vlib_node_t *n = vlib_get_node (vm, rt->node_index);
{
for (j = 0; j < tr->count; j++)
{
+
w = vlib_worker_threads + worker_thread_index++;
err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
w, 0);
if (err)
- clib_error_report (err);
+ clib_unix_error ("%U, thread %s init on cpu %d failed",
+ format_clib_error, err, tr->name, 0);
}
}
else
{
uword c;
- /* *INDENT-OFF* */
clib_bitmap_foreach (c, tr->coremask) {
w = vlib_worker_threads + worker_thread_index++;
err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
w, c);
if (err)
- clib_error_report (err);
- }
- /* *INDENT-ON* */
+ clib_unix_error ("%U, thread %s init on cpu %d failed",
+ format_clib_error, err, tr->name, c);
+ }
}
}
vlib_worker_thread_barrier_sync (vm);
+ {
+ clib_error_t *err;
+ err = vlib_call_init_exit_functions (
+ vm, &vgm->num_workers_change_function_registrations, 1 /* call_once */,
+ 1 /* is_global */);
+ if (err)
+ clib_error_report (err);
+ }
vlib_worker_thread_barrier_release (vm);
return 0;
}
vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
vm_clone->error_main.counters_last_clear = old_counters_all_clear;
+ for (j = 0; j < vec_len (nm_clone->next_frames); j++)
+ {
+ vlib_next_frame_t *nf = &nm_clone->next_frames[j];
+ if ((nf->flags & VLIB_FRAME_IS_ALLOCATED) && nf->frame != NULL)
+ {
+ vlib_frame_t *f = nf->frame;
+ nf->frame = NULL;
+ vlib_frame_free (vm_clone, f);
+ }
+ }
+
vec_free (nm_clone->next_frames);
nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
CLIB_CACHE_LINE_BYTES);
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_resize (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_resize (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
u32 thread_index = vlib_get_thread_index ();
vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index,
sizeof (thread_index));
+ vlib_worker_flush_pending_rpc_requests (vlib_get_main ());
}
/* Wait until main thread asks for barrier */
return;
}
+void
+vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm)
+{
+ vlib_main_t *vm_global = vlib_get_first_main ();
+
+ ASSERT (vm != vm_global);
+
+ clib_spinlock_lock_if_init (&vm_global->pending_rpc_lock);
+ vec_append (vm_global->pending_rpc_requests, vm->pending_rpc_requests);
+ vec_reset_length (vm->pending_rpc_requests);
+ clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock);
+}
+
void
vlib_worker_thread_fn (void *arg)
{
};
/* *INDENT-ON* */
+extern clib_march_fn_registration
+ *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations;
+extern clib_march_fn_registration
+ *vlib_frame_queue_dequeue_fn_march_fn_registrations;
u32
vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
{
vlib_thread_main_t *tm = vlib_get_thread_main ();
+ vlib_main_t *vm = vlib_get_main ();
vlib_frame_queue_main_t *fqm;
vlib_frame_queue_t *fq;
+ vlib_node_t *node;
int i;
u32 num_threads;
vec_add2 (tm->frame_queue_mains, fqm, 1);
+ node = vlib_get_node (vm, fqm->node_index);
+ ASSERT (node);
+ if (node->aux_offset)
+ {
+ fqm->frame_queue_dequeue_fn =
+ CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_with_aux_fn);
+ }
+ else
+ {
+ fqm->frame_queue_dequeue_fn =
+ CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_fn);
+ }
+
fqm->node_index = node_index;
fqm->frame_queue_nelts = frame_queue_nelts;