2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #ifndef included_vlib_threads_h
16 #define included_vlib_threads_h
18 #include <vlib/main.h>
19 #include <vppinfra/callback.h>
21 #include <linux/sched.h>
23 #include <sys/sched.h>
24 #endif /* __linux__ */
26 void vlib_set_thread_name (char *name);
28 /* arg is actually a vlib__thread_t * */
29 typedef void (vlib_thread_function_t) (void *arg);
31 typedef struct vlib_thread_registration_
33 /* constructor generated list of thread registrations */
34 struct vlib_thread_registration_ *next;
36 /* config parameters */
39 vlib_thread_function_t *function;
43 int no_data_structure_clone;
44 u32 frame_queue_nelts;
46 /* All threads of this type run on pthreads */
50 } vlib_thread_registration_t;
52 #define VLIB_LOG2_THREAD_STACK_SIZE (21)
53 #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE)
57 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
62 STRUCT_MARK (end_of_reset);
64 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
65 u32 buffer_index[VLIB_FRAME_SIZE];
66 u32 aux_data[VLIB_FRAME_SIZE];
68 vlib_frame_queue_elt_t;
72 /* First cache line */
73 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
74 volatile u32 *wait_at_barrier;
75 volatile u32 *workers_at_barrier;
77 /* Second Cache Line */
78 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
81 void (*thread_function) (void *);
82 void *thread_function_arg;
84 elog_track_t elog_track;
86 vlib_thread_registration_t *registration;
88 u64 barrier_sync_count;
89 u8 barrier_elog_enabled;
90 const char *barrier_caller;
91 const char *barrier_context;
92 volatile u32 *node_reforks_required;
93 volatile u32 wait_before_barrier;
94 volatile u32 workers_before_barrier;
95 volatile u32 done_work_before_barrier;
102 } vlib_worker_thread_t;
104 extern vlib_worker_thread_t *vlib_worker_threads;
109 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
110 vlib_frame_queue_elt_t *elts;
111 u64 vector_threshold;
115 /* modified by enqueue side */
116 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
119 /* modified by dequeue side */
120 CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
125 struct vlib_frame_queue_main_t_;
126 typedef u32 (vlib_frame_queue_dequeue_fn_t) (
127 vlib_main_t *vm, struct vlib_frame_queue_main_t_ *fqm);
128 typedef struct vlib_frame_queue_main_t_
131 u32 frame_queue_nelts;
133 vlib_frame_queue_t **vlib_frame_queues;
135 /* for frame queue tracing */
136 frame_queue_trace_t *frame_queue_traces;
137 frame_queue_nelt_counter_t *frame_queue_histogram;
138 vlib_frame_queue_dequeue_fn_t *frame_queue_dequeue_fn;
139 } vlib_frame_queue_main_t;
146 } vlib_process_signal_event_mt_args_t;
148 /* Called early, in thread 0's context */
149 clib_error_t *vlib_thread_init (vlib_main_t * vm);
151 void vlib_worker_thread_node_runtime_update (void);
153 void vlib_create_worker_threads (vlib_main_t * vm, int n,
154 void (*thread_function) (void *));
156 void vlib_worker_thread_init (vlib_worker_thread_t * w);
157 u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts);
159 /* Check for a barrier sync request every 30ms */
160 #define BARRIER_SYNC_DELAY (0.030000)
163 /* long barrier timeout, for gdb... */
164 #define BARRIER_SYNC_TIMEOUT (600.1)
166 #define BARRIER_SYNC_TIMEOUT (1.0)
169 #define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);}
171 void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm,
172 const char *func_name);
173 void vlib_worker_thread_barrier_release (vlib_main_t * vm);
174 u8 vlib_worker_thread_barrier_held (void);
175 void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm);
176 void vlib_worker_thread_node_refork (void);
178 * Wait until each of the workers has been once around the track
180 void vlib_worker_wait_one_loop (void);
182 * Flush worker's pending rpc requests to main thread's rpc queue
184 void vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm);
186 static_always_inline uword
187 vlib_get_thread_index (void)
189 return __os_thread_index;
193 vlib_smp_unsafe_warning (void)
197 if (vlib_get_thread_index ())
198 fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__);
203 __foreach_vlib_main_helper (vlib_main_t *ii, vlib_main_t **p, int checks)
206 u32 index = ii - (vlib_main_t *) 0;
208 if (index >= vec_len (vlib_global_main.vlib_mains))
211 *p = vm = vlib_global_main.vlib_mains[index];
212 ASSERT (!checks || index == 0 || vm->parked_at_barrier == 1);
216 #define foreach_vlib_main__(checks) \
217 for (vlib_main_t *ii = 0, *this_vlib_main; \
218 __foreach_vlib_main_helper (ii, &this_vlib_main, checks); ii++) \
221 #define foreach_vlib_main() foreach_vlib_main__ (1)
223 #define foreach_sched_policy_posix \
224 _ (SCHED_OTHER, OTHER, "other") \
225 _ (SCHED_FIFO, FIFO, "fifo") \
226 _ (SCHED_RR, RR, "rr")
227 #define foreach_sched_policy_linux \
228 _ (SCHED_BATCH, BATCH, "batch") \
229 _ (SCHED_IDLE, IDLE, "idle")
232 #define foreach_sched_policy \
233 foreach_sched_policy_posix foreach_sched_policy_linux
235 #define foreach_sched_policy foreach_sched_policy_posix
236 #endif /* __linux__ */
240 #define _(v,f,s) SCHED_POLICY_##f = v,
248 /* Link list of registrations, built by constructors */
249 vlib_thread_registration_t *next;
251 /* Vector of registrations, w/ non-data-structure clones at the top */
252 vlib_thread_registration_t **registrations;
254 uword *thread_registrations_by_name;
256 vlib_worker_thread_t *worker_threads;
260 /* Number of vlib_main / vnet_main clones */
263 /* Number of thread stacks to create */
266 /* Number of pthreads */
269 /* Number of threads */
272 /* Number of cores to skip, must match the core mask */
275 /* Thread prefix name */
278 /* main thread lcore */
281 /* Bitmap of available CPU cores */
282 uword *cpu_core_bitmap;
284 /* Bitmap of available CPU sockets (NUMA nodes) */
285 uword *cpu_socket_bitmap;
287 /* Worker handoff queues */
288 vlib_frame_queue_main_t *frame_queue_mains;
290 /* worker thread initialization barrier */
291 volatile u32 worker_thread_release;
293 /* scheduling policy */
296 /* scheduling policy priority */
299 /* NUMA-bound heap size */
300 uword numa_heap_size;
302 } vlib_thread_main_t;
304 extern vlib_thread_main_t vlib_thread_main;
306 #include <vlib/global_funcs.h>
308 #define VLIB_REGISTER_THREAD(x,...) \
309 __VA_ARGS__ vlib_thread_registration_t x; \
310 static void __vlib_add_thread_registration_##x (void) \
311 __attribute__((__constructor__)) ; \
312 static void __vlib_add_thread_registration_##x (void) \
314 vlib_thread_main_t * tm = &vlib_thread_main; \
318 static void __vlib_rm_thread_registration_##x (void) \
319 __attribute__((__destructor__)) ; \
320 static void __vlib_rm_thread_registration_##x (void) \
322 vlib_thread_main_t * tm = &vlib_thread_main; \
323 VLIB_REMOVE_FROM_LINKED_LIST (tm->next, &x, next); \
325 __VA_ARGS__ vlib_thread_registration_t x
330 return vlib_thread_main.n_vlib_mains - 1;
334 vlib_get_worker_thread_index (u32 worker_index)
336 return worker_index + 1;
340 vlib_get_worker_index (u32 thread_index)
342 return thread_index - 1;
346 vlib_get_current_worker_index ()
348 return vlib_get_thread_index () - 1;
352 vlib_worker_thread_barrier_check (void)
354 if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
356 vlib_global_main_t *vgm = vlib_get_global_main ();
357 vlib_main_t *vm = vlib_get_main ();
358 u32 thread_index = vm->thread_index;
359 f64 t = vlib_time_now (vm);
361 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
362 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
363 vm->clib_time.last_cpu_time, 0 /* enter */ );
365 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
367 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
368 ELOG_TYPE_DECLARE (e) = {
369 .format = "barrier-wait-thread-%d",
378 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track);
379 ed->thread_index = thread_index;
384 vm = vlib_get_main ();
385 vm->parked_at_barrier = 1;
387 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
388 while (*vlib_worker_threads->wait_at_barrier)
392 * Recompute the offset from thread-0 time.
393 * Note that vlib_time_now adds vm->time_offset, so
394 * clear it first. Save the resulting idea of "now", to
395 * see how well we're doing. See show_clock_command_fn(...)
399 vm->time_offset = 0.0;
400 now = vlib_time_now (vm);
401 vm->time_offset = vgm->vlib_mains[0]->time_last_barrier_release - now;
402 vm->time_last_barrier_release = vlib_time_now (vm);
406 vm->parked_at_barrier = 0;
407 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
409 if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required))
411 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
413 t = vlib_time_now (vm) - t;
414 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
415 ELOG_TYPE_DECLARE (e) = {
416 .format = "barrier-refork-thread-%d",
425 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
427 ed->thread_index = thread_index;
430 vlib_worker_thread_node_refork ();
431 clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
433 while (*vlib_worker_threads->node_reforks_required)
436 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
438 t = vlib_time_now (vm) - t;
439 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
440 ELOG_TYPE_DECLARE (e) = {
441 .format = "barrier-released-thread-%d: %dus",
442 .format_args = "i4i4",
451 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, w->elog_track);
452 ed->thread_index = thread_index;
453 ed->duration = (int) (1000000.0 * t);
456 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
457 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
458 vm->clib_time.last_cpu_time, 1 /* leave */ );
462 always_inline vlib_main_t *
463 vlib_get_worker_vlib_main (u32 worker_index)
466 vlib_thread_main_t *tm = &vlib_thread_main;
467 ASSERT (worker_index < tm->n_vlib_mains - 1);
468 vm = vlib_get_main_by_index (worker_index + 1);
474 vlib_thread_is_main_w_barrier (void)
476 return (!vlib_num_workers ()
477 || ((vlib_get_thread_index () == 0
478 && vlib_worker_threads->wait_at_barrier[0])));
481 u8 *vlib_thread_stack_init (uword thread_index);
482 extern void *rpc_call_main_thread_cb_fn;
485 vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
487 void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size);
488 void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id);
489 vlib_thread_main_t *vlib_get_thread_main_not_inline (void);
492 * Force workers sync from within worker
494 * Must be paired with @ref vlib_workers_continue
496 void vlib_workers_sync (void);
498 * Release barrier after workers sync
500 void vlib_workers_continue (void);
502 #endif /* included_vlib_threads_h */
505 * fd.io coding-style-patch-verification: ON
508 * eval: (c-set-style "gnu")