2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #ifndef included_vlib_threads_h
16 #define included_vlib_threads_h
18 #include <vlib/main.h>
20 vlib_main_t **vlib_mains;
23 vlib_set_thread_name (char *name);
25 /* arg is actually a vlib__thread_t * */
26 typedef void (vlib_thread_function_t) (void * arg);
28 typedef struct vlib_thread_registration_ {
29 /* constructor generated list of thread registrations */
30 struct vlib_thread_registration_ * next;
32 /* config parameters */
35 vlib_thread_function_t * function;
39 int no_data_structure_clone;
40 /* All threads of this type run on pthreads */
44 } vlib_thread_registration_t;
47 * Frames have their cpu / vlib_main_t index in the low-order N bits
48 * Make VLIB_MAX_CPUS a power-of-two, please...
51 #define VLIB_MAX_CPUS 256
53 #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */
54 #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK)
56 #define VLIB_LOG2_THREAD_STACK_SIZE (20)
57 #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE)
60 VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME,
61 } vlib_frame_queue_msg_type_t;
69 /* 256 * 4 = 1024 bytes, even mult of cache line size */
70 u32 buffer_index[VLIB_FRAME_SIZE];
72 /* Pad to a cache line boundary */
73 u8 pad[CLIB_CACHE_LINE_BYTES - 4 * sizeof(u32)];
74 } vlib_frame_queue_elt_t;
77 /* First cache line */
78 volatile u32 *wait_at_barrier;
79 volatile u32 *workers_at_barrier;
80 u8 pad0[CLIB_CACHE_LINE_BYTES - (2 * sizeof (u32 *))];
82 /* Second Cache Line */
85 void (*thread_function)(void *);
86 void * thread_function_arg;
88 elog_track_t elog_track;
90 vlib_thread_registration_t *registration;
95 } vlib_worker_thread_t;
97 vlib_worker_thread_t *vlib_worker_threads;
105 u32 enqueue_full_events;
106 u32 enqueue_efd_discards;
107 u8 pad2[CLIB_CACHE_LINE_BYTES
109 - (4 * sizeof(u64))];
117 u64 vector_threshold;
118 u8 pad4[CLIB_CACHE_LINE_BYTES
119 - (6 * sizeof(u64))];
121 /* dequeue hint to enqueue side */
122 volatile u64 head_hint;
123 u8 pad5 [CLIB_CACHE_LINE_BYTES - sizeof(u64)];
125 /* read-only, constant, shared */
126 vlib_frame_queue_elt_t *elts;
128 } vlib_frame_queue_t;
130 vlib_frame_queue_t **vlib_frame_queues;
132 /* Called early, in thread 0's context */
133 clib_error_t * vlib_thread_init (vlib_main_t * vm);
135 vlib_worker_thread_t * vlib_alloc_thread (vlib_main_t * vm);
137 int vlib_frame_queue_enqueue (vlib_main_t *vm, u32 node_runtime_index,
138 u32 frame_queue_index, vlib_frame_t *frame,
139 vlib_frame_queue_msg_type_t type);
141 int vlib_frame_queue_dequeue (int thread_id,
143 vlib_node_main_t *nm);
145 u64 dispatch_node (vlib_main_t * vm,
146 vlib_node_runtime_t * node,
147 vlib_node_type_t type,
148 vlib_node_state_t dispatch_state,
149 vlib_frame_t * frame,
150 u64 last_time_stamp);
152 u64 dispatch_pending_node (vlib_main_t * vm,
153 vlib_pending_frame_t * p,
154 u64 last_time_stamp);
156 void vlib_worker_thread_node_runtime_update(void);
158 void vlib_create_worker_threads (vlib_main_t *vm, int n,
159 void (*thread_function)(void *));
161 void vlib_worker_thread_init (vlib_worker_thread_t * w);
163 /* Check for a barrier sync request every 30ms */
164 #define BARRIER_SYNC_DELAY (0.030000)
167 /* long barrier timeout, for gdb... */
168 #define BARRIER_SYNC_TIMEOUT (600.1)
170 #define BARRIER_SYNC_TIMEOUT (1.0)
173 void vlib_worker_thread_barrier_sync(vlib_main_t *vm);
174 void vlib_worker_thread_barrier_release(vlib_main_t *vm);
176 always_inline void vlib_smp_unsafe_warning (void)
180 if (os_get_cpu_number())
181 fformat(stderr, "%s: SMP unsafe warning...\n", __FUNCTION__);
186 VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0,
187 VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX,
190 void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
192 static inline void vlib_worker_thread_barrier_check (void)
194 if (PREDICT_FALSE(*vlib_worker_threads->wait_at_barrier))
196 clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
197 while (*vlib_worker_threads->wait_at_barrier)
199 clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
203 #define foreach_vlib_main(body) \
205 vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \
208 if (vec_len (vlib_mains) == 0) \
209 vec_add1 (__vlib_mains, &vlib_global_main); \
212 for (ii = 0; ii < vec_len (vlib_mains); ii++) \
214 this_vlib_main = vlib_mains[ii]; \
215 if (this_vlib_main) \
216 vec_add1 (__vlib_mains, this_vlib_main); \
220 for (ii = 0; ii < vec_len (__vlib_mains); ii++) \
222 this_vlib_main = __vlib_mains[ii]; \
223 /* body uses this_vlib_main... */ \
226 vec_free (__vlib_mains); \
230 /* Early-Fast-Discard (EFD) */
231 #define VLIB_EFD_DISABLED 0
232 #define VLIB_EFD_DISCARD_ENABLED (1 << 0)
233 #define VLIB_EFD_MONITOR_ENABLED (1 << 1)
235 #define VLIB_EFD_DEF_WORKER_HI_THRESH_PCT 90
237 /* EFD worker thread settings */
238 typedef struct vlib_efd_t {
248 /* Link list of registrations, built by constructors */
249 vlib_thread_registration_t * next;
251 /* Vector of registrations, w/ non-data-structure clones at the top */
252 vlib_thread_registration_t ** registrations;
254 uword * thread_registrations_by_name;
256 vlib_worker_thread_t * worker_threads;
258 /* thread / cpu / io thread parameters */
259 u32 main_thread_is_io_node;
262 * Launch all threads as pthreads,
263 * not eal_rte_launch (strict affinity) threads
267 /* Number of vlib_main / vnet_main clones */
270 /* Number of thread stacks to create */
273 /* Number of pthreads */
276 /* Number of DPDK eal threads */
279 /* Number of cores to skip, must match the core mask */
282 /* Thread prefix name */
285 /* main thread lcore */
288 /* Bitmap of available CPU cores */
289 uword * cpu_core_bitmap;
291 /* Bitmap of available CPU sockets (NUMA nodes) */
292 uword * cpu_socket_bitmap;
296 } vlib_thread_main_t;
298 vlib_thread_main_t vlib_thread_main;
300 #define VLIB_REGISTER_THREAD(x,...) \
301 __VA_ARGS__ vlib_thread_registration_t x; \
302 static void __vlib_add_thread_registration_##x (void) \
303 __attribute__((__constructor__)) ; \
304 static void __vlib_add_thread_registration_##x (void) \
306 vlib_thread_main_t * tm = &vlib_thread_main; \
310 __VA_ARGS__ vlib_thread_registration_t x
312 #endif /* included_vlib_threads_h */