extern jvpp_main_t jvpp_main __attribute__((aligned (64)));
static_always_inline u32 vppjni_get_context_id(jvpp_main_t * jm) {
- return __sync_add_and_fetch(&jm->context_id, 1);
+ return clib_atomic_add_fetch(&jm->context_id, 1);
}
static_always_inline void vppjni_lock(jvpp_main_t * jm, u32 tag) {
- while (__sync_lock_test_and_set(&jm->lock, 1))
+ while (clib_atomic_test_and_set(&jm->lock))
;
jm->tag = tag;
}
}
sess->deleted = 1;
- clib_smp_atomic_add (&am->fa_session_total_deactivations, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_deactivations, 1);
clib_mem_set_heap (oldheap);
}
vec_validate (pw->fa_session_dels_by_sw_if_index, sw_if_index);
clib_mem_set_heap (oldheap);
pw->fa_session_dels_by_sw_if_index[sw_if_index]++;
- clib_smp_atomic_add (&am->fa_session_total_dels, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_dels, 1);
}
always_inline int
vec_validate (pw->fa_session_adds_by_sw_if_index, sw_if_index);
clib_mem_set_heap (oldheap);
pw->fa_session_adds_by_sw_if_index[sw_if_index]++;
- clib_smp_atomic_add (&am->fa_session_total_adds, 1);
+ clib_atomic_fetch_add (&am->fa_session_total_adds, 1);
return sess;
}
if (PREDICT_FALSE (xd->lockp != 0))
{
queue_id = queue_id % xd->tx_q_used;
- while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
+ while (clib_atomic_test_and_set (xd->lockp[queue_id]))
/* zzzz */
queue_id = (queue_id + 1) % xd->tx_q_used;
}
}
if (PREDICT_FALSE (xd->lockp != 0))
- *xd->lockp[queue_id] = 0;
+ clib_atomic_release (xd->lockp[queue_id]);
if (PREDICT_FALSE (n_sent < 0))
{
ioam_path_map_t *path;
u8 k, i;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
for (k = 0; k < trace_record->num_nodes; k++)
path[k].state_up = 0;
}
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
}
always_inline void
u16 size_of_traceopt_per_node;
u16 size_of_all_traceopts;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
}
}
end:
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
}
always_inline int
ioam_path_map_t *path = NULL;
ioam_analyse_trace_record *trace_record;
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
trace_data = &data->trace_data;
(u32) ((sum + delay) / (data->seqno_data.rx_packets + 1));
}
DONE:
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
return 0;
}
ip6_ioam_analyse_hbh_e2e (ioam_analyser_data_t * data,
ioam_e2e_packet_t * e2e, u16 len)
{
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
ioam_analyze_seqno (&data->seqno_data,
(u64) clib_net_to_host_u32 (e2e->e2e_data));
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
+
return 0;
}
data->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
CLIB_CACHE_LINE_BYTES);
- *(data->writer_lock) = 0;
+
+ clib_atomic_release (data->writer_lock);
trace_data = &(data->trace_data);
for (j = 0; j < IOAM_MAX_PATHS_PER_FLOW; j++)
ip6_address_t * src, ip6_address_t * dst,
u16 src_port, u16 dst_port)
{
- while (__sync_lock_test_and_set (record->writer_lock, 1))
+ while (clib_atomic_test_and_set (record->writer_lock))
;
int field_index = 0;
*(record->chached_data_list) = *record;
record->chached_data_list->chached_data_list = NULL;
- *(record->writer_lock) = 0;
+ clib_atomic_release (record->writer_lock);
return offset;
}
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter += p_len0;
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
- while (__sync_lock_test_and_set (data1->writer_lock, 1))
+ while (clib_atomic_test_and_set (data1->writer_lock))
;
data1->pkt_counter++;
data1->bytes_counter += p_len1;
- *(data1->writer_lock) = 0;
+ clib_atomic_release (data1->writer_lock);
}
else if (error0 == 0)
{
pkts_failed++;
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter += p_len0;
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
}
else if (error1 == 0)
{
pkts_failed++;
data1 = ioam_analyse_get_data_from_flow_id (flow_id1);
- while (__sync_lock_test_and_set (data1->writer_lock, 1))
+ while (clib_atomic_test_and_set (data1->writer_lock))
;
data1->pkt_counter++;
data1->bytes_counter += p_len1;
- *(data1->writer_lock) = 0;
+ clib_atomic_release (data1->writer_lock);
}
else
pkts_failed += 2;
{
pkts_analysed++;
data0 = ioam_analyse_get_data_from_flow_id (flow_id0);
- while (__sync_lock_test_and_set (data0->writer_lock, 1))
+ while (clib_atomic_test_and_set (data0->writer_lock))
;
data0->pkt_counter++;
data0->bytes_counter +=
clib_net_to_host_u16 (ip60->payload_length);
- *(data0->writer_lock) = 0;
+ clib_atomic_release (data0->writer_lock);
}
else
pkts_failed++;
pot_profile = pot_profile_get_active ();
ret = pot_validate (pot_profile, cumulative, random);
- while (__sync_lock_test_and_set (data->writer_lock, 1))
+ while (clib_atomic_test_and_set (data->writer_lock))
;
(0 == ret) ? (data->pot_data.sfc_validated_count++) :
(data->pot_data.sfc_invalidated_count++);
- *(data->writer_lock) = 0;
+ clib_atomic_release (data->writer_lock);
return 0;
}
*/
for (i = 0; i < vec_len (thread_index); i++)
{
- while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1))
+ while (clib_atomic_test_and_set (em->lockp[thread_index[i]]))
;
em->buffer_per_thread[thread_index[i]] =
vec_pop (vec_buffer_indices);
- *em->lockp[thread_index[i]] = 0;
+ clib_atomic_release (em->lockp[thread_index[i]]);
}
/* Send the buffers */
from = vlib_frame_vector_args (F); \
n_left_from = (F)->n_vectors; \
next_index = (N)->cached_next_index; \
- while (__sync_lock_test_and_set ((EM)->lockp[(VM)->thread_index], 1)); \
+ while (clib_atomic_test_and_set ((EM)->lockp[(VM)->thread_index])); \
my_buf = ioam_export_get_my_buffer (EM, (VM)->thread_index); \
my_buf->touched_at = vlib_time_now (VM); \
while (n_left_from > 0) \
lb_main_t lb_main;
-#define lb_get_writer_lock() do {} while(__sync_lock_test_and_set (lb_main.writer_lock, 1))
-#define lb_put_writer_lock() lb_main.writer_lock[0] = 0
+#define lb_get_writer_lock() do {} while(clib_atomic_test_and_set (lb_main.writer_lock))
+#define lb_put_writer_lock() clib_atomic_release (lb_main.writer_lock)
static void lb_as_stack (lb_as_t *as);
void
map_ip4_reass_free(map_ip4_reass_t *r, u32 **pi_to_drop);
-#define map_ip4_reass_lock() while (__sync_lock_test_and_set(map_main.ip4_reass_lock, 1)) {}
+#define map_ip4_reass_lock() while (clib_atomic_test_and_set (map_main.ip4_reass_lock)) {}
#define map_ip4_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip4_reass_lock = 0;} while(0)
static_always_inline void
void
map_ip6_reass_free(map_ip6_reass_t *r, u32 **pi_to_drop);
-#define map_ip6_reass_lock() while (__sync_lock_test_and_set(map_main.ip6_reass_lock, 1)) {}
+#define map_ip6_reass_lock() while (clib_atomic_test_and_set (map_main.ip6_reass_lock)) {}
#define map_ip6_reass_unlock() do {CLIB_MEMORY_BARRIER(); *map_main.ip6_reass_lock = 0;} while(0)
int
map_domain_counter_lock (map_main_t *mm)
{
if (mm->counter_lock)
- while (__sync_lock_test_and_set(mm->counter_lock, 1))
+ while (clib_atomic_test_and_set (mm->counter_lock))
/* zzzz */ ;
}
static inline void
map_domain_counter_unlock (map_main_t *mm)
{
if (mm->counter_lock)
- *mm->counter_lock = 0;
+ clib_atomic_release (mm->counter_lock);
}
{
if (!dm->sessions[i + user_offset].in_port)
{
- if (__sync_bool_compare_and_swap
+ if (clib_atomic_bool_cmp_and_swap
(&dm->sessions[i + user_offset].in_port, 0, in_port))
{
dm->sessions[i + user_offset].out.as_u64 = out->as_u64;
dm->sessions[i + user_offset].state = SNAT_SESSION_UNKNOWN;
dm->sessions[i + user_offset].expire = 0;
- __sync_add_and_fetch (&dm->ses_num, 1);
+ clib_atomic_add_fetch (&dm->ses_num, 1);
return &dm->sessions[i + user_offset];
}
}
always_inline void
snat_det_ses_close (snat_det_map_t * dm, snat_det_session_t * ses)
{
- if (__sync_bool_compare_and_swap (&ses->in_port, ses->in_port, 0))
+ if (clib_atomic_bool_cmp_and_swap (&ses->in_port, ses->in_port, 0))
{
ses->out.as_u64 = 0;
- __sync_add_and_fetch (&dm->ses_num, -1);
+ clib_atomic_add_fetch (&dm->ses_num, -1);
}
}
msg.ring_index = ring - mq->rings;
msg.elt_index = ring->tail;
ring->tail = (ring->tail + 1) % ring->nitems;
- __sync_fetch_and_add (&ring->cursize, 1);
+ clib_atomic_fetch_add (&ring->cursize, 1);
return msg;
}
msg.ring_index = ring - mq->rings;
msg.elt_index = ring->tail;
ring->tail = (ring->tail + 1) % ring->nitems;
- __sync_fetch_and_add (&ring->cursize, 1);
+ clib_atomic_fetch_add (&ring->cursize, 1);
break;
}
return msg;
/* for now, expect messages to be processed in order */
ASSERT (0);
}
- __sync_fetch_and_sub (&ring->cursize, 1);
+ clib_atomic_fetch_sub (&ring->cursize, 1);
}
static int
return;
}
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->owner_pid = my_pid;
always_inline void
ssvm_lock_non_recursive (ssvm_shared_header_t * h, u32 tag)
{
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->tag = tag;
/* Atomically increase the queue length */
ASSERT (cursize + total_copy_bytes <= nitems);
- __sync_fetch_and_add (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_add (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_copy_bytes);
- __sync_fetch_and_sub (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_drop_bytes);
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
return total_drop_bytes;
}
svm_fifo_dequeue_drop_all (svm_fifo_t * f)
{
f->head = f->tail;
- __sync_fetch_and_sub (&f->cursize, f->cursize);
+ clib_atomic_fetch_sub (&f->cursize, f->cursize);
}
int
f->head = (f->head + fs[0].len) % f->nitems;
total_drop_bytes = fs[0].len;
}
- __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
}
u32
always_inline void
svm_fifo_unset_event (svm_fifo_t * f)
{
- __sync_lock_release (&f->has_event);
+ clib_atomic_release (&f->has_event);
}
static inline void
tail->total_length_not_including_first_buffer;
next_segment:
- __sync_add_and_fetch (&tail->n_add_refs, 1);
+ clib_atomic_add_fetch (&tail->n_add_refs, 1);
if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
{
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
p = hash_get (vlib_buffer_state_validation_hash, b);
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
hash_set (vlib_buffer_state_validation_hash, b, expected);
ASSERT (fq);
- new_tail = __sync_add_and_fetch (&fq->tail, 1);
+ new_tail = clib_atomic_add_fetch (&fq->tail, 1);
/* Wait until a ring slot is available */
while (new_tail >= fq->head + fq->nelts)
{
/* Initial barrier sync, for both worker and i/o threads */
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
while (*vlib_worker_threads->wait_at_barrier)
;
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
}
}
VLIB_EARLY_CONFIG_FUNCTION (cpu_config, "cpu");
-#if !defined (__x86_64__) && !defined (__i386__) && !defined (__aarch64__) && !defined (__powerpc64__) && !defined(__arm__)
-void
-__sync_fetch_and_add_8 (void)
-{
- fformat (stderr, "%s called\n", __FUNCTION__);
- abort ();
-}
-
-void
-__sync_add_and_fetch_8 (void)
-{
- fformat (stderr, "%s called\n", __FUNCTION__);
- abort ();
-}
-#endif
-
void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
void
vnet_main_fixup (vlib_fork_fixup_t which)
/* Do per thread rebuilds in parallel */
refork_needed = 1;
- clib_smp_atomic_add (vlib_worker_threads->node_reforks_required,
- (vec_len (vlib_mains) - 1));
+ clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
+ (vec_len (vlib_mains) - 1));
now = vlib_time_now (vm);
t_update_main = now - vm->barrier_epoch;
}
ed->thread_index = thread_index;
}
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
if (CLIB_DEBUG > 0)
{
vm = vlib_get_main ();
;
if (CLIB_DEBUG > 0)
vm->parked_at_barrier = 0;
- clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+ clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required))
{
}
vlib_worker_thread_node_refork ();
- clib_smp_atomic_add (vlib_worker_threads->node_reforks_required,
- -1);
+ clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
+ -1);
while (*vlib_worker_threads->node_reforks_required)
;
}
fq = fqm->vlib_frame_queues[index];
ASSERT (fq);
- new_tail = __sync_add_and_fetch (&fq->tail, 1);
+ new_tail = clib_atomic_add_fetch (&fq->tail, 1);
/* Wait until a ring slot is available */
while (new_tail >= fq->head_hint + fq->nelts)
if (cjm->enable == 0)
return;
- new_tail = __sync_add_and_fetch (&cjm->tail, 1);
+ new_tail = clib_atomic_add_fetch (&cjm->tail, 1);
r = (cj_record_t *) & (cjm->records[new_tail & (cjm->num_records - 1)]);
r->time = vlib_time_now (cjm->vlib_main);
hash >>= t->log2_nbuckets;
- while (__sync_lock_test_and_set (t->writer_lock, 1))
+ while (clib_atomic_test_and_set (t->writer_lock))
;
/* First elt in the bucket? */
static_always_inline int
vhost_user_vring_try_lock (vhost_user_intf_t * vui, u32 qid)
{
- return __sync_lock_test_and_set (vui->vring_locks[qid], 1);
+ return clib_atomic_test_and_set (vui->vring_locks[qid]);
}
/**
static_always_inline void
vhost_user_vring_unlock (vhost_user_intf_t * vui, u32 qid)
{
- *vui->vring_locks[qid] = 0;
+ clib_atomic_release (vui->vring_locks[qid]);
}
static_always_inline void
{
if (dm->cache_lock)
{
- while (__sync_lock_test_and_set (dm->cache_lock, 1))
+ while (clib_atomic_test_and_set (dm->cache_lock))
;
}
}
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b0, -sizeof (erspan_t2_t));
erspan_t2_t *h0 = vlib_buffer_get_current (b0);
- u32 seq_num = clib_smp_atomic_add (>0->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (>0->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h0->seq_num = clib_host_to_net_u32 (seq_num);
h0->t2_u64 = hdr;
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b1, -sizeof (erspan_t2_t));
erspan_t2_t *h1 = vlib_buffer_get_current (b1);
- u32 seq_num = clib_smp_atomic_add (>1->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (>1->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h1->seq_num = clib_host_to_net_u32 (seq_num);
h1->t2_u64 = hdr;
/* Encap GRE seq# and ERSPAN type II header */
vlib_buffer_advance (b0, -sizeof (erspan_t2_t));
erspan_t2_t *h0 = vlib_buffer_get_current (b0);
- u32 seq_num = clib_smp_atomic_add (>0->gre_sn->seq_num, 1);
+ u32 seq_num = clib_atomic_fetch_add (>0->gre_sn->seq_num, 1);
u64 hdr = clib_host_to_net_u64 (ERSPAN_HDR2);
h0->seq_num = clib_host_to_net_u32 (seq_num);
h0->t2_u64 = hdr;
vnet_interface_counter_lock (vnet_interface_main_t * im)
{
if (im->sw_if_counter_lock)
- while (__sync_lock_test_and_set (im->sw_if_counter_lock, 1))
+ while (clib_atomic_test_and_set (im->sw_if_counter_lock))
/* zzzz */ ;
}
vnet_interface_counter_unlock (vnet_interface_main_t * im)
{
if (im->sw_if_counter_lock)
- *im->sw_if_counter_lock = 0;
+ clib_atomic_release (im->sw_if_counter_lock);
}
void vnet_pcap_drop_trace_filter_add_del (u32 error_index, int is_add);
else if (new_leaf_dst_address_bits >=
ply->dst_address_bits_of_leaves[i])
{
- __sync_val_compare_and_swap (&ply->leaves[i], old_leaf, new_leaf);
+ clib_atomic_cmp_and_swap (&ply->leaves[i], old_leaf, new_leaf);
ASSERT (ply->leaves[i] == new_leaf);
ply->dst_address_bits_of_leaves[i] = new_leaf_dst_address_bits;
ply->n_non_empty_leafs += ip4_fib_mtrie_leaf_is_non_empty (ply, i);
old_ply->dst_address_bits_of_leaves[i] =
a->dst_address_length;
- __sync_val_compare_and_swap (&old_ply->leaves[i], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[i], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[i] == new_leaf);
old_ply->n_non_empty_leafs +=
/* Refetch since ply_create may move pool. */
old_ply = pool_elt_at_index (ip4_ply_pool, old_ply_index);
- __sync_val_compare_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[dst_byte] == new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
* the new one */
old_ply->dst_address_bits_of_leaves[slot] =
a->dst_address_length;
- __sync_val_compare_and_swap (&old_ply->leaves[slot],
- old_leaf, new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[slot],
+ old_leaf, new_leaf);
ASSERT (old_ply->leaves[slot] == new_leaf);
}
else
ply_base_len);
new_ply = get_next_ply_for_leaf (m, new_leaf);
- __sync_val_compare_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
+ clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
+ new_leaf);
ASSERT (old_ply->leaves[dst_byte] == new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
}
t = pool_elt_at_index (vcm->tables, table->classify_table_index);
- while (__sync_lock_test_and_set (t->writer_lock, 1))
+ while (clib_atomic_test_and_set (t->writer_lock))
;
for (i = 0; i < t->nbuckets; i++)
bi0 = ~0;
}
- *(t->writer_lock) = 0;
+ clib_atomic_release (t->writer_lock);
return f;
}
{
mfib_itf_flags_t old_flags;
- old_flags = __sync_fetch_and_or(&mfi->mfi_flags,
- MFIB_ITF_FLAG_SIGNAL_PRESENT);
+ old_flags = clib_atomic_fetch_or(&mfi->mfi_flags,
+ MFIB_ITF_FLAG_SIGNAL_PRESENT);
if (!(old_flags & MFIB_ITF_FLAG_SIGNAL_PRESENT))
{
static inline void
mfib_signal_lock_aquire (void)
{
- while (__sync_lock_test_and_set (&mfib_signal_pending.mip_lock, 1))
+ while (clib_atomic_test_and_set (&mfib_signal_pending.mip_lock))
;
}
static inline void
mfib_signal_lock_release (void)
{
- mfib_signal_pending.mip_lock = 0;
+ clib_atomic_release(&mfib_signal_pending.mip_lock);
}
#define MFIB_SIGNAL_CRITICAL_SECTION(_body) \
mfs = pool_elt_at_index(mfib_signal_pool, si);
mfi = mfib_itf_get(mfs->mfs_itf);
mfi->mfi_si = INDEX_INVALID;
- __sync_fetch_and_and(&mfi->mfi_flags,
- ~MFIB_ITF_FLAG_SIGNAL_PRESENT);
+ clib_atomic_fetch_and(&mfi->mfi_flags,
+ ~MFIB_ITF_FLAG_SIGNAL_PRESENT);
vl_mfib_signal_send_one(reg, context, mfs);
pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
if (PREDICT_FALSE (pif->lockp != 0))
- while (__sync_lock_test_and_set (pif->lockp, 1))
+ while (clib_atomic_test_and_set (pif->lockp))
;
while (n_left > 0)
vlib_buffer_free (vm, vlib_frame_args (frame), n_buffers);
if (PREDICT_FALSE (pif->lockp != 0))
- *pif->lockp = 0;
+ clib_atomic_release (pif->lockp);
+
return n_buffers;
}
{
stream_session_t *s;
- __sync_fetch_and_add (&ecm->tx_total, sp->bytes_sent);
- __sync_fetch_and_add (&ecm->rx_total, sp->bytes_received);
+ clib_atomic_fetch_add (&ecm->tx_total, sp->bytes_sent);
+ clib_atomic_fetch_add (&ecm->rx_total, sp->bytes_received);
s = session_get_from_handle_if_valid (sp->vpp_session_handle);
if (s)
vec_delete (connections_this_batch, 1, i);
i--;
- __sync_fetch_and_add (&ecm->ready_connections, -1);
+ clib_atomic_fetch_add (&ecm->ready_connections, -1);
}
else
{
}
vec_add1 (ecm->connection_index_by_thread[thread_index], session_index);
- __sync_fetch_and_add (&ecm->ready_connections, 1);
+ clib_atomic_fetch_add (&ecm->ready_connections, 1);
if (ecm->ready_connections == ecm->expected_connections)
{
ecm->run_test = ECHO_CLIENTS_RUNNING;
static_always_inline
void vlib_refcount_lock (volatile u32 *counter_lock)
{
- while (__sync_lock_test_and_set (counter_lock, 1))
+ while (clib_atomic_test_and_set (counter_lock))
;
}
static_always_inline
void vlib_refcount_unlock (volatile u32 *counter_lock)
{
- *counter_lock = 0;
+ clib_atomic_release(counter_lock);
}
void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size);
if (release_hint)
l->release_hint++;
- while (__sync_lock_test_and_set (&l->lock, 1))
+ while (clib_atomic_test_and_set (&l->lock))
/* zzzz */ ;
l->tag = tag;
l->thread_index = thread_index;
tw_timer_template.c
tw_timer_template.h
types.h
+ atomics.h
unix.h
valgrind.h
valloc.h
--- /dev/null
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Copyright (c) 2018 Arm Limited. and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_clib_atomics_h
+#define included_clib_atomics_h
+
+/* Legacy __sync builtins */
+
+/* Full Barrier */
+#define clib_atomic_fetch_add(a, b) __sync_fetch_and_add(a, b)
+#define clib_atomic_fetch_sub(a, b) __sync_fetch_and_sub(a, b)
+#define clib_atomic_fetch_and(a, b) __sync_fetch_and_and(a, b)
+#define clib_atomic_fetch_xor(a, b) __sync_fetch_and_xor(a, b)
+#define clib_atomic_fetch_or(a, b) __sync_fetch_and_or(a, b)
+#define clib_atomic_fetch_nand(a, b) __sync_fetch_nand(a, b)
+
+#define clib_atomic_add_fetch(a, b) __sync_add_and_fetch(a, b)
+#define clib_atomic_sub_fetch(a, b) __sync_sub_and_fetch(a, b)
+#define clib_atomic_and_fetch(a, b) __sync_and_and_fetch(a, b)
+#define clib_atomic_xor_fetch(a, b) __sync_xor_and_fetch(a, b)
+#define clib_atomic_or_fetch(a, b) __sync_or_and_fetch(a, b)
+#define clib_atomic_nand_fetch(a, b) __sync_nand_and_fetch(a, b)
+
+#define clib_atomic_cmp_and_swap(addr,old,new) __sync_val_compare_and_swap(addr, old, new)
+#define clib_atomic_bool_cmp_and_swap(addr,old,new) __sync_bool_compare_and_swap(addr, old, new)
+
+/*Accquire Barrier*/
+#define clib_atomic_test_and_set(a) __sync_lock_test_and_set(a, 1)
+/*Release Barrier*/
+#define clib_atomic_release(a) __sync_lock_release(a)
+
+#endif /* included_clib_atomics_h */
#endif
#include <vppinfra/types.h>
+#include <vppinfra/atomics.h>
/* Global DEBUG flag. Setting this to 1 or 0 turns off
ASSERT (see vppinfra/error.h) & other debugging code. */
elog_lock (elog_main_t * em)
{
if (PREDICT_FALSE (em->lock != 0))
- while (__sync_lock_test_and_set (em->lock, 1))
+ while (clib_atomic_test_and_set (em->lock))
;
}
ASSERT (is_pow2 (vec_len (em->event_ring)));
if (em->lock)
- ei = clib_smp_atomic_add (&em->n_total_events, 1);
+ ei = clib_atomic_fetch_add (&em->n_total_events, 1);
else
ei = em->n_total_events++;
static_always_inline void
clib_spinlock_lock (clib_spinlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->lock))
CLIB_PAUSE ();
CLIB_LOCK_DBG (p);
}
always_inline void
clib_rwlock_reader_lock (clib_rwlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->n_readers_lock))
CLIB_PAUSE ();
(*p)->n_readers += 1;
if ((*p)->n_readers == 1)
{
- while (__sync_lock_test_and_set (&(*p)->writer_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->writer_lock))
CLIB_PAUSE ();
}
CLIB_MEMORY_BARRIER ();
ASSERT ((*p)->n_readers > 0);
CLIB_LOCK_DBG_CLEAR (p);
- while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->n_readers_lock))
CLIB_PAUSE ();
(*p)->n_readers -= 1;
always_inline void
clib_rwlock_writer_lock (clib_rwlock_t * p)
{
- while (__sync_lock_test_and_set (&(*p)->writer_lock, 1))
+ while (clib_atomic_test_and_set (&(*p)->writer_lock))
CLIB_PAUSE ();
CLIB_LOCK_DBG (p);
}
ASSERT (mm->flags & CLIB_MAPLOG_FLAG_INIT);
- my_record_index = __sync_fetch_and_add (&mm->next_record_index, 1);
+ my_record_index = clib_atomic_fetch_add (&mm->next_record_index, 1);
/* Time to unmap and create a new logfile? */
if (PREDICT_FALSE ((my_record_index & (mm->file_size_in_records - 1)) == 0))
return;
}
- while (__sync_lock_test_and_set (&h->lock, 1))
+ while (clib_atomic_test_and_set (&h->lock))
;
h->owner_cpu = my_cpu;
#include <vppinfra/cache.h>
#include <vppinfra/os.h> /* for os_panic */
-#define clib_smp_compare_and_swap(addr,new,old) __sync_val_compare_and_swap(addr,old,new)
#define clib_smp_swap(addr,new) __sync_lock_test_and_set(addr,new)
-#define clib_smp_atomic_add(addr,increment) __sync_fetch_and_add(addr,increment)
#if defined (i386) || defined (__x86_64__)
#define clib_smp_pause() do { asm volatile ("pause"); } while (0)
Code Review / vpp.git / commitdiff