#include <signal.h>
#include <math.h>
#include <vppinfra/format.h>
+#include <vppinfra/linux/sysfs.h>
#include <vlib/vlib.h>
#include <vlib/threads.h>
DECLARE_CJ_GLOBAL_LOG;
-#define FRAME_QUEUE_NELTS 32
+#define FRAME_QUEUE_NELTS 64
u32
vl (void *p)
* imapacts observed timings.
*/
-#ifdef BARRIER_TRACING
- /*
- * Output of barrier tracing can be to syslog or elog as suits
- */
-#ifdef BARRIER_TRACING_ELOG
-static u32
-elog_id_for_msg_name (const char *msg_name)
+u32
+elog_global_id_for_msg_name (const char *msg_name)
{
uword *p, r;
static uword *h;
return r;
}
- /*
- * elog Barrier trace functions, which are nulled out if BARRIER_TRACING isn't
- * defined
- */
-
static inline void
barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
{
+ if (!vlib_worker_threads->barrier_elog_enabled)
+ return;
+
/* *INDENT-OFF* */
ELOG_TYPE_DECLARE (e) =
{
- .format = "barrier <%d#%s(O:%dus:%dus)(%dus)",
- .format_args = "i4T4i4i4i4",
+ .format = "bar-trace-%s-#%d",
+ .format_args = "T4i4",
};
/* *INDENT-ON* */
struct
{
- u32 count, caller, t_entry, t_open, t_closed;
+ u32 caller, count, t_entry, t_open, t_closed;
} *ed = 0;
ed = ELOG_DATA (&vlib_global_main.elog_main, e);
ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
- ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller);
+ ed->caller = elog_global_id_for_msg_name
+ (vlib_worker_threads[0].barrier_caller);
ed->t_entry = (int) (1000000.0 * t_entry);
ed->t_open = (int) (1000000.0 * t_open);
ed->t_closed = (int) (1000000.0 * t_closed);
static inline void
barrier_trace_sync_rec (f64 t_entry)
{
+ if (!vlib_worker_threads->barrier_elog_enabled)
+ return;
+
/* *INDENT-OFF* */
ELOG_TYPE_DECLARE (e) =
{
- .format = "barrier <%d(%dus)%s",
- .format_args = "i4i4T4",
+ .format = "bar-syncrec-%s-#%d",
+ .format_args = "T4i4",
};
/* *INDENT-ON* */
struct
{
- u32 depth, t_entry, caller;
+ u32 caller, depth;
} *ed = 0;
ed = ELOG_DATA (&vlib_global_main.elog_main, e);
ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
- ed->t_entry = (int) (1000000.0 * t_entry);
- ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller);
+ ed->caller = elog_global_id_for_msg_name
+ (vlib_worker_threads[0].barrier_caller);
}
static inline void
barrier_trace_release_rec (f64 t_entry)
{
+ if (!vlib_worker_threads->barrier_elog_enabled)
+ return;
+
/* *INDENT-OFF* */
ELOG_TYPE_DECLARE (e) =
{
- .format = "barrier (%dus)%d>",
- .format_args = "i4i4",
+ .format = "bar-relrrec-#%d",
+ .format_args = "i4",
};
/* *INDENT-ON* */
struct
{
- u32 t_entry, depth;
+ u32 depth;
} *ed = 0;
ed = ELOG_DATA (&vlib_global_main.elog_main, e);
- ed->t_entry = (int) (1000000.0 * t_entry);
ed->depth = (int) vlib_worker_threads[0].recursion_level;
}
static inline void
barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
{
+ if (!vlib_worker_threads->barrier_elog_enabled)
+ return;
+
/* *INDENT-OFF* */
ELOG_TYPE_DECLARE (e) =
{
- .format = "barrier (%dus){%d}(C:%dus)#%d>",
+ .format = "bar-rel-#%d-e%d-u%d-t%d",
.format_args = "i4i4i4i4",
};
/* *INDENT-ON* */
struct
{
- u32 t_entry, t_update_main, t_closed_total, count;
+ u32 count, t_entry, t_update_main, t_closed_total;
} *ed = 0;
ed = ELOG_DATA (&vlib_global_main.elog_main, e);
/* Reset context for next trace */
vlib_worker_threads[0].barrier_context = NULL;
}
-#else
-char barrier_trace[65536];
-char *btp = barrier_trace;
-
- /*
- * syslog Barrier trace functions, which are nulled out if BARRIER_TRACING
- * isn't defined
- */
-
-
-static inline void
-barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
-{
- btp += sprintf (btp, "<%u#%s",
- (unsigned int) vlib_worker_threads[0].barrier_sync_count,
- vlib_worker_threads[0].barrier_caller);
-
- if (vlib_worker_threads[0].barrier_context)
- {
- btp += sprintf (btp, "[%s]", vlib_worker_threads[0].barrier_context);
-
- }
-
- btp += sprintf (btp, "(O:%dus:%dus)(%dus):",
- (int) (1000000.0 * t_entry),
- (int) (1000000.0 * t_open), (int) (1000000.0 * t_closed));
-
-}
-
-static inline void
-barrier_trace_sync_rec (f64 t_entry)
-{
- btp += sprintf (btp, "<%u(%dus)%s:",
- (int) vlib_worker_threads[0].recursion_level - 1,
- (int) (1000000.0 * t_entry),
- vlib_worker_threads[0].barrier_caller);
-}
-
-static inline void
-barrier_trace_release_rec (f64 t_entry)
-{
- btp += sprintf (btp, ":(%dus)%u>", (int) (1000000.0 * t_entry),
- (int) vlib_worker_threads[0].recursion_level);
-}
-
-static inline void
-barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
-{
-
- btp += sprintf (btp, ":(%dus)", (int) (1000000.0 * t_entry));
- if (t_update_main > 0)
- {
- btp += sprintf (btp, "{%dus}", (int) (1000000.0 * t_update_main));
- }
-
- btp += sprintf (btp, "(C:%dus)#%u>",
- (int) (1000000.0 * t_closed_total),
- (int) vlib_worker_threads[0].barrier_sync_count);
-
- /* Dump buffer to syslog, and reset for next trace */
- fformat (stderr, "BTRC %s\n", barrier_trace);
- btp = barrier_trace;
- vlib_worker_threads[0].barrier_context = NULL;
-}
-#endif
-#else
-
- /* Null functions for default case where barrier tracing isn't used */
-static inline void
-barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
-{
-}
-
-static inline void
-barrier_trace_sync_rec (f64 t_entry)
-{
-}
-
-static inline void
-barrier_trace_release_rec (f64 t_entry)
-{
-}
-
-static inline void
-barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
-{
-}
-#endif
uword
os_get_nthreads (void)
}
/* grab cpu for main thread */
- if (!tm->main_lcore)
+ if (tm->main_lcore == ~0)
{
- tm->main_lcore = clib_bitmap_first_set (avail_cpu);
+ /* if main-lcore is not set, we try to use lcore 1 */
+ if (clib_bitmap_get (avail_cpu, 1))
+ tm->main_lcore = 1;
+ else
+ tm->main_lcore = clib_bitmap_first_set (avail_cpu);
if (tm->main_lcore == (u8) ~ 0)
return clib_error_return (0, "no available cpus to be used for the"
" main thread");
w = vlib_worker_threads;
w->thread_mheap = clib_mem_get_heap ();
w->thread_stack = vlib_thread_stacks[0];
- w->lcore_id = tm->main_lcore;
+ w->cpu_id = tm->main_lcore;
w->lwp = syscall (SYS_gettid);
w->thread_id = pthread_self ();
tm->n_vlib_mains = 1;
return rv;
}
+static void
+vlib_get_thread_core_socket (vlib_worker_thread_t * w, unsigned cpu_id)
+{
+ const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
+ u8 *p = 0;
+ int core_id = -1, socket_id = -1;
+
+ p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
+ clib_sysfs_read ((char *) p, "%d", &core_id);
+ vec_reset_length (p);
+ p =
+ format (p, "%s%u/topology/physical_package_id%c", sys_cpu_path, cpu_id,
+ 0);
+ clib_sysfs_read ((char *) p, "%d", &socket_id);
+ vec_free (p);
+
+ w->core_id = core_id;
+ w->socket_id = socket_id;
+}
+
static clib_error_t *
-vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned lcore_id)
+vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
{
vlib_thread_main_t *tm = &vlib_thread_main;
void *(*fp_arg) (void *) = fp;
- w->lcore_id = lcore_id;
+ w->cpu_id = cpu_id;
+ vlib_get_thread_core_socket (w, cpu_id);
if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads)
- return tm->cb.vlib_launch_thread_cb (fp, (void *) w, lcore_id);
+ return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
else
{
pthread_t worker;
cpu_set_t cpuset;
CPU_ZERO (&cpuset);
- CPU_SET (lcore_id, &cpuset);
+ CPU_SET (cpu_id, &cpuset);
if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
return clib_error_return_unix (0, "pthread_create");
u32 n_vlib_mains = tm->n_vlib_mains;
u32 worker_thread_index;
u8 *main_heap = clib_mem_get_per_cpu_heap ();
- mheap_t *main_heap_header = mheap_header (main_heap);
vec_reset_length (vlib_worker_threads);
vlib_set_thread_name ((char *) w->name);
}
- /*
- * Truth of the matter: we always use at least two
- * threads. So, make the main heap thread-safe
- * and make the event log thread-safe.
- */
- main_heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
vm->elog_main.lock =
clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES);
vm->elog_main.lock[0] = 0;
for (i = 0; i < vec_len (tm->registrations); i++)
{
vlib_node_main_t *nm, *nm_clone;
- vlib_buffer_main_t *bm_clone;
vlib_buffer_free_list_t *fl_clone, *fl_orig;
vlib_buffer_free_list_t *orig_freelist_pool;
int k;
vlib_node_t *n;
vec_add2 (vlib_worker_threads, w, 1);
+ /* Currently unused, may not really work */
if (tr->mheap_size)
- w->thread_mheap =
- mheap_alloc (0 /* use VM */ , tr->mheap_size);
+ {
+#if USE_DLMALLOC == 0
+ w->thread_mheap =
+ mheap_alloc (0 /* use VM */ , tr->mheap_size);
+#else
+ w->thread_mheap = create_mspace (tr->mheap_size,
+ 0 /* unlocked */ );
+#endif
+ }
else
w->thread_mheap = main_heap;
/* Fork vlib_global_main et al. Look for bugs here */
oldheap = clib_mem_set_heap (w->thread_mheap);
- vm_clone = clib_mem_alloc (sizeof (*vm_clone));
+ vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
+ CLIB_CACHE_LINE_BYTES);
clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
vm_clone->thread_index = worker_thread_index;
vm_clone->heap_base = w->thread_mheap;
- vm_clone->mbuf_alloc_list = 0;
+ vm_clone->heap_aligned_base = (void *)
+ (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
vm_clone->init_functions_called =
hash_create (0, /* value bytes */ 0);
+ vm_clone->pending_rpc_requests = 0;
+ vec_validate (vm_clone->pending_rpc_requests, 0);
+ _vec_len (vm_clone->pending_rpc_requests) = 0;
memset (&vm_clone->random_buffer, 0,
sizeof (vm_clone->random_buffer));
nm = &vlib_mains[0]->node_main;
nm_clone = &vm_clone->node_main;
/* fork next frames array, preserving node runtime indices */
- nm_clone->next_frames = vec_dup (nm->next_frames);
+ nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
+ CLIB_CACHE_LINE_BYTES);
for (j = 0; j < vec_len (nm_clone->next_frames); j++)
{
vlib_next_frame_t *nf = &nm_clone->next_frames[j];
n++;
}
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
- vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+ vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
+ CLIB_CACHE_LINE_BYTES);
vec_foreach (rt,
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
{
}
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
- vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
+ vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
+ CLIB_CACHE_LINE_BYTES);
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
vlib_node_t *n = vlib_get_node (vm, rt->node_index);
n->runtime_data_bytes));
}
- nm_clone->processes = vec_dup (nm->processes);
+ nm_clone->processes = vec_dup_aligned (nm->processes,
+ CLIB_CACHE_LINE_BYTES);
/* zap the (per worker) frame freelists, etc */
nm_clone->frame_sizes = 0;
clib_mem_set_heap (oldheap);
vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES);
- vm_clone->error_main.counters =
- vec_dup (vlib_mains[0]->error_main.counters);
- vm_clone->error_main.counters_last_clear =
- vec_dup (vlib_mains[0]->error_main.counters_last_clear);
+ vm_clone->error_main.counters = vec_dup_aligned
+ (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
+ vm_clone->error_main.counters_last_clear = vec_dup_aligned
+ (vlib_mains[0]->error_main.counters_last_clear,
+ CLIB_CACHE_LINE_BYTES);
/* Fork the vlib_buffer_main_t free lists, etc. */
- bm_clone = vec_dup (vm_clone->buffer_main);
- vm_clone->buffer_main = bm_clone;
-
- orig_freelist_pool = bm_clone->buffer_free_list_pool;
- bm_clone->buffer_free_list_pool = 0;
+ orig_freelist_pool = vm_clone->buffer_free_list_pool;
+ vm_clone->buffer_free_list_pool = 0;
/* *INDENT-OFF* */
pool_foreach (fl_orig, orig_freelist_pool,
({
- pool_get_aligned (bm_clone->buffer_free_list_pool,
+ pool_get_aligned (vm_clone->buffer_free_list_pool,
fl_clone, CLIB_CACHE_LINE_BYTES);
ASSERT (fl_orig - orig_freelist_pool
- == fl_clone - bm_clone->buffer_free_list_pool);
+ == fl_clone - vm_clone->buffer_free_list_pool);
fl_clone[0] = fl_orig[0];
fl_clone->buffers = 0;
{
vec_add2 (vlib_worker_threads, w, 1);
if (tr->mheap_size)
- w->thread_mheap =
- mheap_alloc (0 /* use VM */ , tr->mheap_size);
+ {
+#if USE_DLMALLOC == 0
+ w->thread_mheap =
+ mheap_alloc (0 /* use VM */ , tr->mheap_size);
+#else
+ w->thread_mheap =
+ create_mspace (tr->mheap_size, 0 /* locked */ );
+#endif
+ }
else
w->thread_mheap = main_heap;
w->thread_stack =
nm_clone = &vm_clone->node_main;
vec_free (nm_clone->next_frames);
- nm_clone->next_frames = vec_dup (nm->next_frames);
+ nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
+ CLIB_CACHE_LINE_BYTES);
for (j = 0; j < vec_len (nm_clone->next_frames); j++)
{
/* re-clone internal nodes */
old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
- vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+ vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
+ CLIB_CACHE_LINE_BYTES);
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
{
/* re-clone input nodes */
old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
- vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
+ vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
+ CLIB_CACHE_LINE_BYTES);
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
vec_free (old_rt);
- nm_clone->processes = vec_dup (nm->processes);
+ nm_clone->processes = vec_dup_aligned (nm->processes,
+ CLIB_CACHE_LINE_BYTES);
}
void
uword *p;
vlib_thread_main_t *tm = &vlib_thread_main;
u8 *name;
- u64 coremask;
uword *bitmap;
u32 count;
tm->n_thread_stacks = 1; /* account for main thread */
tm->sched_policy = ~0;
tm->sched_priority = ~0;
+ tm->main_lcore = ~0;
tr = tm->next;
;
else if (unformat (input, "skip-cores %u", &tm->skip_cores))
;
- else if (unformat (input, "coremask-%s %llx", &name, &coremask))
- {
- p = hash_get_mem (tm->thread_registrations_by_name, name);
- if (p == 0)
- return clib_error_return (0, "no such thread type '%s'", name);
-
- tr = (vlib_thread_registration_t *) p[0];
-
- if (tr->use_pthreads)
- return clib_error_return (0,
- "coremask cannot be set for '%s' threads",
- name);
-
- tr->coremask = clib_bitmap_set_multiple
- (tr->coremask, 0, coremask, BITS (coremask));
- tr->count = clib_bitmap_count_set_bits (tr->coremask);
- }
- else if (unformat (input, "corelist-%s %U", &name, unformat_bitmap_list,
- &bitmap))
+ else if (unformat (input, "coremask-%s %U", &name,
+ unformat_bitmap_mask, &bitmap) ||
+ unformat (input, "corelist-%s %U", &name,
+ unformat_bitmap_list, &bitmap))
{
p = hash_get_mem (tm->thread_registrations_by_name, name);
if (p == 0)
/* Enforce minimum barrier open time to minimize packet loss */
ASSERT (vm->barrier_no_close_before <= (now + BARRIER_MINIMUM_OPEN_LIMIT));
- while ((now = vlib_time_now (vm)) < vm->barrier_no_close_before)
- ;
+ while (1)
+ {
+ now = vlib_time_now (vm);
+ /* Barrier hold-down timer expired? */
+ if (now >= vm->barrier_no_close_before)
+ break;
+ if ((vm->barrier_no_close_before - now)
+ > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
+ {
+ clib_warning ("clock change: would have waited for %.4f seconds",
+ (vm->barrier_no_close_before - now));
+ break;
+ }
+ }
/* Record time of closure */
t_open = now - vm->barrier_epoch;
vm->barrier_epoch = now;
}
+void vlib_stat_segment_lock (void) __attribute__ ((weak));
+void
+vlib_stat_segment_lock (void)
+{
+}
+
+void vlib_stat_segment_unlock (void) __attribute__ ((weak));
+void
+vlib_stat_segment_unlock (void)
+{
+}
+
void
vlib_worker_thread_barrier_release (vlib_main_t * vm)
{
/* Update (all) node runtimes before releasing the barrier, if needed */
if (vm->need_vlib_worker_thread_node_runtime_update)
{
+ /*
+ * Lock stat segment here, so we's safe when
+ * rebuilding the stat segment node clones from the
+ * stat thread...
+ */
+ vlib_stat_segment_lock ();
+
/* Do stats elements on main thread */
worker_thread_node_runtime_update_internal ();
vm->need_vlib_worker_thread_node_runtime_update = 0;
os_panic ();
}
}
+ vlib_stat_segment_unlock ();
}
t_closed_total = now - vm->barrier_epoch;
if (frame_queue_nelts == 0)
frame_queue_nelts = FRAME_QUEUE_NELTS;
+ ASSERT (frame_queue_nelts >= 8);
+
vec_add2 (tm->frame_queue_mains, fqm, 1);
fqm->node_index = node_index;
+ fqm->frame_queue_nelts = frame_queue_nelts;
+ fqm->queue_hi_thresh = frame_queue_nelts - 2;
vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1);
+ vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
_vec_len (fqm->vlib_frame_queues) = 0;
for (i = 0; i < tm->n_vlib_mains; i++)
{
+ vlib_frame_queue_per_thread_data_t *ptd;
fq = vlib_frame_queue_alloc (frame_queue_nelts);
vec_add1 (fqm->vlib_frame_queues, fq);
+
+ ptd = vec_elt_at_index (fqm->per_thread_data, i);
+ vec_validate (ptd->handoff_queue_elt_by_thread_index,
+ tm->n_vlib_mains - 1);
+ vec_validate_init_empty (ptd->congested_handoff_queue_by_thread_index,
+ tm->n_vlib_mains - 1,
+ (vlib_frame_queue_t *) (~0));
}
return (fqm - tm->frame_queue_mains);
}
-
int
vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb)
{
return 0;
}
+void
+vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
+ args)
+{
+ ASSERT (vlib_get_thread_index () == 0);
+ vlib_process_signal_event (vlib_get_main (), args->node_index,
+ args->type_opaque, args->data);
+}
+
+void *rpc_call_main_thread_cb_fn;
+
+void
+vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
+{
+ if (rpc_call_main_thread_cb_fn)
+ {
+ void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
+ (*fp) (callback, args, arg_size);
+ }
+ else
+ clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
+}
+
clib_error_t *
threads_init (vlib_main_t * vm)
{
Code Review / vpp.git / blobdiff