2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 Copyright (c) 2001-2005 Eliot Dresselhaus
18 Permission is hereby granted, free of charge, to any person obtaining
19 a copy of this software and associated documentation files (the
20 "Software"), to deal in the Software without restriction, including
21 without limitation the rights to use, copy, modify, merge, publish,
22 distribute, sublicense, and/or sell copies of the Software, and to
23 permit persons to whom the Software is furnished to do so, subject to
24 the following conditions:
26 The above copyright notice and this permission notice shall be
27 included in all copies or substantial portions of the Software.
29 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 #ifndef included_clib_smp_h
39 #define included_clib_smp_h
41 #include <vppinfra/cache.h>
42 #include <vppinfra/os.h> /* for os_panic */
46 /* Per-cpu local heap. */
50 } clib_smp_per_cpu_main_t;
53 /* Number of CPUs used to model current computer. */
56 /* Number of cpus that are done and have exited. */
59 /* Log2 stack and vm (heap) size. */
60 u8 log2_n_per_cpu_stack_bytes, log2_n_per_cpu_vm_bytes;
62 /* Thread local store (TLS) is stored at stack top.
63 Number of 4k pages to allocate for TLS. */
66 /* Per cpus stacks/heaps start at these addresses. */
69 /* Thread-safe global heap. Objects here can be allocated/freed by any cpu. */
72 clib_smp_per_cpu_main_t * per_cpu_mains;
75 extern clib_smp_main_t clib_smp_main;
78 clib_smp_vm_base_for_cpu (clib_smp_main_t * m, uword cpu)
80 return m->vm_base + (cpu << m->log2_n_per_cpu_vm_bytes);
84 clib_smp_stack_top_for_cpu (clib_smp_main_t * m, uword cpu)
86 /* Stack is at top of per cpu VM area. */
87 return clib_smp_vm_base_for_cpu (m, cpu + 1) - ((uword) 1 << m->log2_n_per_cpu_stack_bytes);
91 os_get_cpu_number_inline (void)
93 clib_smp_main_t * m = &clib_smp_main;
97 /* Get any old stack address. */
100 n = ((uword)sp - (uword)m->vm_base) >> m->log2_n_per_cpu_vm_bytes;
102 if (CLIB_DEBUG && m->n_cpus > 0 && n >= m->n_cpus)
105 return n < m->n_cpus ? n : 0;
108 #define clib_smp_compare_and_swap(addr,new,old) __sync_val_compare_and_swap(addr,old,new)
109 #define clib_smp_swap(addr,new) __sync_lock_test_and_set(addr,new)
110 #define clib_smp_atomic_add(addr,increment) __sync_fetch_and_add(addr,increment)
112 #if defined (i386) || defined (__x86_64__)
113 #define clib_smp_pause() do { asm volatile ("pause"); } while (0)
116 #ifndef clib_smp_pause
117 #define clib_smp_pause() do { } while (0)
124 os_sched_yield (void)
128 os_sched_yield (void)
129 { clib_smp_pause (); }
133 CLIB_SMP_LOCK_TYPE_READER,
134 CLIB_SMP_LOCK_TYPE_WRITER,
135 CLIB_SMP_LOCK_TYPE_SPIN,
136 } clib_smp_lock_type_t;
139 CLIB_SMP_LOCK_WAIT_EMPTY,
140 CLIB_SMP_LOCK_WAIT_DONE,
141 CLIB_SMP_LOCK_WAIT_READER,
142 CLIB_SMP_LOCK_WAIT_WRITER,
143 } clib_smp_lock_wait_type_t;
146 typedef u16 clib_smp_quarter_word_t;
147 typedef u32 clib_smp_half_word_t;
149 typedef u8 clib_smp_quarter_word_t;
150 typedef u16 clib_smp_half_word_t;
155 /* FIFO of CPUs (threads) waiting for lock. */
157 clib_smp_quarter_word_t head_index, n_elts;
160 /* Requesting CPU for atomic compare_and_swap instructions.
161 This makes CPUs requesting same header change unique. */
162 clib_smp_quarter_word_t request_cpu;
164 /* Count of readers who have been given read lock.
165 Not applicable for spin locks. */
166 clib_smp_quarter_word_t n_readers_with_lock : BITS (clib_smp_quarter_word_t) - 1;
168 /* Set when writer has been given write lock. Only one of
169 these can happen at a time. */
170 clib_smp_quarter_word_t writer_has_lock : 1;
174 } clib_smp_lock_header_t;
177 clib_smp_lock_header_is_equal (clib_smp_lock_header_t h0, clib_smp_lock_header_t h1)
178 { return h0.as_uword == h1.as_uword; }
181 volatile clib_smp_lock_wait_type_t wait_type;
182 u8 pad[CLIB_CACHE_LINE_BYTES - 1 * sizeof (clib_smp_lock_wait_type_t)];
183 } clib_smp_lock_waiting_fifo_elt_t;
187 clib_smp_lock_header_t header;
189 /* Size of waiting FIFO; equal to max number of threads less one. */
190 u32 n_waiting_fifo_elts;
192 u8 pad[CLIB_CACHE_LINE_BYTES - sizeof (clib_smp_lock_header_t) - sizeof (u32)];
194 clib_smp_lock_waiting_fifo_elt_t waiting_fifo[0];
197 always_inline clib_smp_lock_header_t
198 clib_smp_lock_set_header (clib_smp_lock_t * l, clib_smp_lock_header_t new_hdr, clib_smp_lock_header_t old)
200 clib_smp_lock_header_t cmp;
201 cmp.as_uword = clib_smp_compare_and_swap (&l->header.as_uword, new_hdr.as_uword, old.as_uword);
205 void clib_smp_lock_init (clib_smp_lock_t ** l);
206 void clib_smp_lock_free (clib_smp_lock_t ** l);
207 void clib_smp_lock_slow_path (clib_smp_lock_t * l, uword my_cpu, clib_smp_lock_header_t h, clib_smp_lock_type_t type);
208 void clib_smp_unlock_slow_path (clib_smp_lock_t * l, uword my_cpu, clib_smp_lock_header_t h, clib_smp_lock_type_t type);
211 clib_smp_lock_inline (clib_smp_lock_t * l, clib_smp_lock_type_t type)
213 clib_smp_lock_header_t h0, h1, h2;
214 uword is_reader = type == CLIB_SMP_LOCK_TYPE_READER;
217 /* Null lock means n_cpus <= 1: nothing to lock. */
221 my_cpu = os_get_cpu_number_inline ();
223 while (! h0.writer_has_lock)
225 /* Want to write but there are still readers with lock? */
226 if (type == CLIB_SMP_LOCK_TYPE_WRITER && h0.n_readers_with_lock != 0)
229 if (type == CLIB_SMP_LOCK_TYPE_SPIN)
230 ASSERT_AND_PANIC (h0.waiting_fifo.n_elts == 0);
232 /* Read/write can't proceed when waiting fifo is non-empty. */
233 else if (h0.waiting_fifo.n_elts != 0)
237 h1.request_cpu = my_cpu;
238 h1.writer_has_lock = ! is_reader;
239 h1.n_readers_with_lock += is_reader;
241 /* Try to set head and tail to zero and thereby get the lock. */
242 h2 = clib_smp_lock_set_header (l, h1, h0);
244 /* Compare and swap succeeded? If so, we got the lock. */
245 if (clib_smp_lock_header_is_equal (h2, h0))
248 /* Header for slow path. */
252 clib_smp_lock_slow_path (l, my_cpu, h0, type);
256 clib_smp_unlock_inline (clib_smp_lock_t * l, clib_smp_lock_type_t type)
258 clib_smp_lock_header_t h0, h1;
259 uword is_reader = type == CLIB_SMP_LOCK_TYPE_READER;
262 /* Null means no locking is necessary. */
266 my_cpu = os_get_cpu_number_inline ();
269 /* Should be locked. */
272 ASSERT_AND_PANIC (h0.n_readers_with_lock != 0);
273 ASSERT_AND_PANIC (h0.writer_has_lock == 0);
277 ASSERT_AND_PANIC (h0.n_readers_with_lock == 0);
278 ASSERT_AND_PANIC (h0.writer_has_lock);
281 /* Locked but empty waiting fifo? */
282 while (h0.waiting_fifo.n_elts == 0)
284 /* Try to mark it unlocked. */
287 h1.n_readers_with_lock -= 1;
289 h1.writer_has_lock = 0;
290 h1.request_cpu = my_cpu;
291 h1 = clib_smp_lock_set_header (l, h1, h0);
292 if (clib_smp_lock_header_is_equal (h1, h0))
297 /* Other cpus are waiting. */
298 clib_smp_unlock_slow_path (l, my_cpu, h0, type);
302 clib_smp_lock (clib_smp_lock_t * l)
303 { clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_SPIN); }
306 clib_smp_lock_for_writer (clib_smp_lock_t * l)
307 { clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_WRITER); }
310 clib_smp_lock_for_reader (clib_smp_lock_t * l)
311 { clib_smp_lock_inline (l, CLIB_SMP_LOCK_TYPE_READER); }
314 clib_smp_unlock (clib_smp_lock_t * l)
315 { clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_SPIN); }
318 clib_smp_unlock_for_writer (clib_smp_lock_t * l)
319 { clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_WRITER); }
322 clib_smp_unlock_for_reader (clib_smp_lock_t * l)
323 { clib_smp_unlock_inline (l, CLIB_SMP_LOCK_TYPE_READER); }
325 #define clib_exec_on_global_heap(body) \
327 void * __clib_exec_on_global_heap_saved_heap; \
329 /* Switch to global (thread-safe) heap. */ \
330 __clib_exec_on_global_heap_saved_heap = clib_mem_set_heap (clib_smp_main.global_heap); \
332 /* Execute body. */ \
335 /* Switch back to previous heap. */ \
336 clib_mem_set_heap (__clib_exec_on_global_heap_saved_heap); \
339 uword os_smp_bootstrap (uword n_cpus,
340 void * bootstrap_function,
341 uword bootstrap_function_arg);
343 void clib_smp_init (void);
345 #endif /* included_clib_smp_h */