#define included_vlib_threads_h
#include <vlib/main.h>
+#include <vppinfra/callback.h>
#include <linux/sched.h>
-extern vlib_main_t **vlib_mains;
-
void vlib_set_thread_name (char *name);
/* arg is actually a vlib__thread_t * */
volatile u32 *node_reforks_required;
long lwp;
- int lcore_id;
+ int cpu_id;
+ int core_id;
+ int numa_id;
pthread_t thread_id;
} vlib_worker_thread_t;
#define BARRIER_SYNC_TIMEOUT (1.0)
#endif
-#define vlib_worker_thread_barrier_sync(X) {vlib_worker_threads[0].barrier_caller=__FUNCTION__;vlib_worker_thread_barrier_sync_int(X);}
+#define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);}
-void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm);
+void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm,
+ const char *func_name);
void vlib_worker_thread_barrier_release (vlib_main_t * vm);
+u8 vlib_worker_thread_barrier_held (void);
+void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm);
void vlib_worker_thread_node_refork (void);
+/**
+ * Wait until each of the workers has been once around the track
+ */
+void vlib_worker_wait_one_loop (void);
static_always_inline uword
vlib_get_thread_index (void)
}
}
-typedef enum
+always_inline int
+__foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p)
{
- VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0,
- VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX,
-} vlib_fork_fixup_t;
-
-void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
-
-#define foreach_vlib_main(body) \
-do { \
- vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \
- int ii; \
- \
- for (ii = 0; ii < vec_len (vlib_mains); ii++) \
- { \
- this_vlib_main = vlib_mains[ii]; \
- ASSERT (ii == 0 || \
- this_vlib_main->parked_at_barrier == 1); \
- if (this_vlib_main) \
- vec_add1 (__vlib_mains, this_vlib_main); \
- } \
- \
- for (ii = 0; ii < vec_len (__vlib_mains); ii++) \
- { \
- this_vlib_main = __vlib_mains[ii]; \
- /* body uses this_vlib_main... */ \
- (body); \
- } \
- vec_free (__vlib_mains); \
-} while (0);
+ vlib_main_t *vm;
+ u32 index = ii - (vlib_main_t *) 0;
+
+ if (index >= vec_len (vlib_global_main.vlib_mains))
+ return 0;
+
+ *p = vm = vlib_global_main.vlib_mains[index];
+ ASSERT (index == 0 || vm->parked_at_barrier == 1);
+ return 1;
+}
+
+#define foreach_vlib_main() \
+ for (vlib_main_t *ii = 0, *this_vlib_main; \
+ __foreach_vlib_main_helper (ii, &this_vlib_main); ii++) \
+ if (this_vlib_main)
#define foreach_sched_policy \
_(SCHED_OTHER, OTHER, "other") \
typedef struct
{
clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w,
- unsigned lcore_id);
- clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 lcore);
+ unsigned cpu_id);
+ clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu);
} vlib_thread_callbacks_t;
typedef struct
/* callbacks */
vlib_thread_callbacks_t cb;
int extern_thread_mgmt;
+
+ /* NUMA-bound heap size */
+ uword numa_heap_size;
+
} vlib_thread_main_t;
extern vlib_thread_main_t vlib_thread_main;
{
if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
{
+ vlib_global_main_t *vgm = vlib_get_global_main ();
vlib_main_t *vm = vlib_get_main ();
u32 thread_index = vm->thread_index;
f64 t = vlib_time_now (vm);
+ if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
+ clib_call_callbacks (vm->barrier_perf_callbacks, vm,
+ vm->clib_time.last_cpu_time, 0 /* enter */ );
+
if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
{
vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
u32 thread_index;
} __clib_packed *ed;
- ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
- w->elog_track);
+ ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track);
ed->thread_index = thread_index;
}
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
if (CLIB_DEBUG > 0)
{
vm = vlib_get_main ();
vm->parked_at_barrier = 1;
}
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
while (*vlib_worker_threads->wait_at_barrier)
;
+
+ /*
+ * Recompute the offset from thread-0 time.
+ * Note that vlib_time_now adds vm->time_offset, so
+ * clear it first. Save the resulting idea of "now", to
+ * see how well we're doing. See show_clock_command_fn(...)
+ */
+ {
+ f64 now;
+ vm->time_offset = 0.0;
+ now = vlib_time_now (vm);
+ vm->time_offset = vgm->vlib_mains[0]->time_last_barrier_release - now;
+ vm->time_last_barrier_release = vlib_time_now (vm);
+ }
+
if (CLIB_DEBUG > 0)
vm->parked_at_barrier = 0;
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required))
{
}
vlib_worker_thread_node_refork ();
- clib_smp_atomic_add (vlib_worker_threads->node_reforks_required,
- -1);
+ clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
+ -1);
while (*vlib_worker_threads->node_reforks_required)
;
}
u32 duration;
} __clib_packed *ed;
- ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
- w->elog_track);
+ ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track);
ed->thread_index = thread_index;
ed->duration = (int) (1000000.0 * t);
}
+
+ if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
+ clib_call_callbacks (vm->barrier_perf_callbacks, vm,
+ vm->clib_time.last_cpu_time, 1 /* leave */ );
}
}
vlib_main_t *vm;
vlib_thread_main_t *tm = &vlib_thread_main;
ASSERT (worker_index < tm->n_vlib_mains - 1);
- vm = vlib_mains[worker_index + 1];
+ vm = vlib_get_main_by_index (worker_index + 1);
ASSERT (vm);
return vm;
}
fq = fqm->vlib_frame_queues[index];
ASSERT (fq);
- new_tail = __sync_add_and_fetch (&fq->tail, 1);
+ new_tail = clib_atomic_add_fetch (&fq->tail, 1);
/* Wait until a ring slot is available */
while (new_tail >= fq->head_hint + fq->nelts)
vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
args);
void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size);
+void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id);
+vlib_thread_main_t *vlib_get_thread_main_not_inline (void);
#endif /* included_vlib_threads_h */
Code Review / vpp.git / blobdiff