2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef included_clib_lock_h
17 #define included_clib_lock_h
19 #include <vppinfra/clib.h>
20 #include <vppinfra/atomics.h>
23 #define CLIB_PAUSE() __builtin_ia32_pause ()
24 #elif defined (__aarch64__) || defined (__arm__)
25 #define CLIB_PAUSE() __asm__ ("yield")
31 #define CLIB_LOCK_DBG(_p) \
33 (*_p)->frame_address = __builtin_frame_address (0); \
34 (*_p)->pid = getpid (); \
35 (*_p)->thread_index = os_get_thread_index (); \
37 #define CLIB_LOCK_DBG_CLEAR(_p) \
39 (*_p)->frame_address = 0; \
41 (*_p)->thread_index = 0; \
44 #define CLIB_LOCK_DBG(_p)
45 #define CLIB_LOCK_DBG_CLEAR(_p)
48 #define CLIB_SPINLOCK_IS_LOCKED(_p) (*(_p))->lock
49 #define CLIB_SPINLOCK_ASSERT_LOCKED(_p) ASSERT(CLIB_SPINLOCK_IS_LOCKED((_p)))
51 struct clib_spinlock_s
53 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
62 typedef struct clib_spinlock_s *clib_spinlock_t;
65 clib_spinlock_init (clib_spinlock_t * p)
67 *p = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES);
68 clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES);
72 clib_spinlock_free (clib_spinlock_t * p)
76 clib_mem_free ((void *) *p);
81 static_always_inline void
82 clib_spinlock_lock (clib_spinlock_t * p)
85 while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->lock, &free, 1, 0))
87 /* atomic load limits number of compare_exchange executions */
88 while (clib_atomic_load_relax_n (&(*p)->lock))
90 /* on failure, compare_exchange writes (*p)->lock into free */
96 static_always_inline int
97 clib_spinlock_trylock (clib_spinlock_t * p)
99 if (PREDICT_FALSE (CLIB_SPINLOCK_IS_LOCKED (p)))
101 clib_spinlock_lock (p);
105 static_always_inline void
106 clib_spinlock_lock_if_init (clib_spinlock_t * p)
108 if (PREDICT_FALSE (*p != 0))
109 clib_spinlock_lock (p);
112 static_always_inline int
113 clib_spinlock_trylock_if_init (clib_spinlock_t * p)
115 if (PREDICT_FALSE (*p != 0))
116 return clib_spinlock_trylock (p);
120 static_always_inline void
121 clib_spinlock_unlock (clib_spinlock_t * p)
123 CLIB_LOCK_DBG_CLEAR (p);
124 /* Make sure all reads/writes are complete before releasing the lock */
125 clib_atomic_release (&(*p)->lock);
128 static_always_inline void
129 clib_spinlock_unlock_if_init (clib_spinlock_t * p)
131 if (PREDICT_FALSE (*p != 0))
132 clib_spinlock_unlock (p);
136 * Readers-Writer Lock
139 typedef struct clib_rw_lock_
141 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
142 /* -1 when W lock held, > 0 when R lock held */
152 clib_rwlock_init (clib_rwlock_t * p)
154 *p = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES);
155 clib_memset ((void *) *p, 0, CLIB_CACHE_LINE_BYTES);
159 clib_rwlock_free (clib_rwlock_t * p)
163 clib_mem_free ((void *) *p);
169 clib_rwlock_reader_lock (clib_rwlock_t * p)
174 /* rwlock held by a writer */
175 while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) < 0)
178 while (!clib_atomic_cmp_and_swap_acq_relax_n
179 (&(*p)->rw_cnt, &cnt, cnt + 1, 1));
184 clib_rwlock_reader_unlock (clib_rwlock_t * p)
186 ASSERT ((*p)->rw_cnt > 0);
187 CLIB_LOCK_DBG_CLEAR (p);
188 clib_atomic_fetch_sub_rel (&(*p)->rw_cnt, 1);
192 clib_rwlock_writer_lock (clib_rwlock_t * p)
197 /* rwlock held by writer or reader(s) */
198 while ((cnt = clib_atomic_load_relax_n (&(*p)->rw_cnt)) != 0)
201 while (!clib_atomic_cmp_and_swap_acq_relax_n (&(*p)->rw_cnt, &cnt, -1, 1));
206 clib_rwlock_writer_unlock (clib_rwlock_t * p)
208 CLIB_LOCK_DBG_CLEAR (p);
209 clib_atomic_release (&(*p)->rw_cnt);
215 * fd.io coding-style-patch-verification: ON
218 * eval: (c-set-style "gnu")