-/*
- * Copyright (c) 2020 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Cisco Systems, Inc.
*/
#ifndef included_clib_interrupt_h
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- int n_int;
- uword n_uword_alloc;
+ u32 n_int;
+ u32 uwords_allocated;
+ u32 uwords_used;
+ uword *local;
+ uword *remote;
} clib_interrupt_header_t;
-void clib_interrupt_init (void **data, uword n_interrupts);
-void clib_interrupt_resize (void **data, uword n_interrupts);
+void clib_interrupt_init (void **data, u32 n_interrupts);
+void clib_interrupt_resize (void **data, u32 n_interrupts);
static_always_inline void
clib_interrupt_free (void **data)
return 0;
}
-static_always_inline uword *
-clib_interrupt_get_bitmap (void *d)
-{
- return d + sizeof (clib_interrupt_header_t);
-}
-
-static_always_inline uword *
-clib_interrupt_get_atomic_bitmap (void *d)
-{
- clib_interrupt_header_t *h = d;
- return clib_interrupt_get_bitmap (d) + h->n_uword_alloc;
-}
-
static_always_inline void
clib_interrupt_set (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- bmp += int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- *bmp |= mask;
+ h->local[off] |= bit;
}
static_always_inline void
clib_interrupt_set_atomic (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_atomic_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- bmp += int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- __atomic_fetch_or (bmp, mask, __ATOMIC_RELAXED);
+ __atomic_fetch_or (h->remote + off, bit, __ATOMIC_RELAXED);
}
static_always_inline void
clib_interrupt_clear (void *in, int int_num)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword *abm = clib_interrupt_get_atomic_bitmap (in);
- uword mask = 1ULL << (int_num & (uword_bits - 1));
- uword off = int_num >> log2_uword_bits;
+ clib_interrupt_header_t *h = in;
+ u32 off = int_num >> log2_uword_bits;
+ uword bit = 1ULL << (int_num & pow2_mask (log2_uword_bits));
+ uword *loc = h->local;
+ uword *rem = h->remote;
+ uword v;
- ASSERT (int_num < ((clib_interrupt_header_t *) in)->n_int);
+ ASSERT (int_num < h->n_int);
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp[off] &= ~mask;
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v & ~bit;
}
static_always_inline int
-clib_interrupt_get_next (void *in, int last)
+clib_interrupt_get_next_and_clear (void *in, int last)
{
- uword *bmp = clib_interrupt_get_bitmap (in);
- uword *abm = clib_interrupt_get_atomic_bitmap (in);
clib_interrupt_header_t *h = in;
- uword bmp_uword, off;
+ uword bit, v;
+ uword *loc = h->local;
+ uword *rem = h->remote;
+ u32 off, n_uwords = h->uwords_used;
- ASSERT (last >= -1 && last < h->n_int);
+ ASSERT (last >= -1 && last < (int) h->n_int);
off = (last + 1) >> log2_uword_bits;
- if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
+
+ if (off >= n_uwords)
return -1;
- last -= off << log2_uword_bits;
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp_uword = bmp[off] & ~pow2_mask (last + 1);
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v;
+
+ v &= ~pow2_mask ((last + 1) & pow2_mask (log2_uword_bits));
-next:
- if (bmp_uword)
- return (off << log2_uword_bits) + count_trailing_zeros (bmp_uword);
+ while (v == 0)
+ {
+ if (++off == n_uwords)
+ return -1;
- off++;
+ v = loc[off] | __atomic_exchange_n (rem + off, 0, __ATOMIC_SEQ_CST);
+ loc[off] = v;
+ }
- if (off > h->n_int >> log2_uword_bits || off >= h->n_uword_alloc)
- return -1;
+ bit = get_lowest_set_bit (v);
+ loc[off] &= ~bit;
+ return get_lowest_set_bit_index (bit) + (int) (off << log2_uword_bits);
+}
+
+static_always_inline int
+clib_interrupt_is_any_pending (void *in)
+{
+ clib_interrupt_header_t *h = in;
+ u32 n_uwords = h->uwords_used;
+ uword *loc = h->local;
+ uword *rem = h->remote;
+
+ for (u32 i = 0; i < n_uwords; i++)
+ if (loc[i])
+ return 1;
- bmp[off] |= __atomic_exchange_n (abm + off, 0, __ATOMIC_SEQ_CST);
- bmp_uword = bmp[off];
+ for (u32 i = 0; i < n_uwords; i++)
+ if (rem[i])
+ return 1;
- goto next;
+ return 0;
}
#endif /* included_clib_interrupt_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
Code Review / vpp.git / blobdiff