fss_chunk_free_list_push (fifo_segment_slice_t * fss, u32 fl_index,
svm_fifo_chunk_t * c)
{
+ clib_spinlock_lock (&fss->chunk_lock);
c->next = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = c;
+ clib_spinlock_unlock (&fss->chunk_lock);
}
static void
svm_fifo_chunk_t * head,
svm_fifo_chunk_t * tail)
{
+ clib_spinlock_lock (&fss->chunk_lock);
tail->next = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = head;
+ clib_spinlock_unlock (&fss->chunk_lock);
}
static svm_fifo_chunk_t *
ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index));
+ clib_spinlock_lock (&fss->chunk_lock);
+
if (!fss->free_chunks[fl_index])
- return 0;
+ {
+ clib_spinlock_unlock (&fss->chunk_lock);
+ return 0;
+ }
c = fss->free_chunks[fl_index];
fss->free_chunks[fl_index] = c->next;
+ clib_spinlock_unlock (&fss->chunk_lock);
+
return c;
}
}
}
+static inline uword
+fss_fl_chunk_bytes (fifo_segment_slice_t * fss)
+{
+ return clib_atomic_load_relax_n (&fss->n_fl_chunk_bytes);
+}
+
+static inline void
+fss_fl_chunk_bytes_add (fifo_segment_slice_t * fss, uword size)
+{
+ clib_atomic_fetch_add_relax (&fss->n_fl_chunk_bytes, size);
+}
+
+static inline void
+fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size)
+{
+ clib_atomic_fetch_sub_relax (&fss->n_fl_chunk_bytes, size);
+}
+
/**
* Initialize fifo segment shared header
*/
}
done:
- fss->n_fl_chunk_bytes -= n_alloc;
+ fss_fl_chunk_bytes_sub (fss, n_alloc);
fsh_cached_bytes_sub (fsh, n_alloc);
return first;
}
fss_chunk_free_list_push_list (fss, fl_index, head, tail);
fss->num_chunks[fl_index] += batch_size;
- fss->n_fl_chunk_bytes += batch_size * rounded_data_size;
+ fss_fl_chunk_bytes_add (fss, batch_size * rounded_data_size);
fsh_cached_bytes_add (fsh, batch_size * rounded_data_size);
fsh_free_bytes_sub (fsh, size);
if (c)
{
c->next = 0;
- fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
+ fss_fl_chunk_bytes_sub (fss, fs_freelist_index_to_size (fl_index));
fsh_cached_bytes_sub (fsh, fs_freelist_index_to_size (fl_index));
}
else
}
/* Failed to allocate larger chunk, try to allocate multi-chunk
* that is close to what was actually requested */
- if (data_bytes <= fss->n_fl_chunk_bytes)
+ if (data_bytes <= fss_fl_chunk_bytes (fss))
{
c = fs_try_alloc_multi_chunk (fsh, fss, data_bytes);
if (c)
goto done;
}
}
- if (data_bytes <= fss->n_fl_chunk_bytes + n_free)
+ if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
{
u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE;
- batch = (data_bytes - fss->n_fl_chunk_bytes) / min_size;
+ batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
batch = clib_min (batch + 1, n_free / min_size);
if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
{
fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
u32 data_bytes)
{
- u32 fl_index;
+ u32 fl_index, min_size;
svm_fifo_chunk_t *c;
svm_fifo_t *f = 0;
- u32 min_size;
min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
fl_index = fs_freelist_for_size (min_size);
if (!fss_chunk_fl_index_is_valid (fss, fl_index))
return 0;
- clib_spinlock_lock (&fss->chunk_lock);
-
f = fsh_try_alloc_fifo_hdr (fsh, fss);
if (!f)
- goto done;
+ return 0;
c = fsh_try_alloc_chunk (fsh, fss, min_size);
- if (c)
- {
- f->start_chunk = c;
- while (c->next)
- c = c->next;
- f->end_chunk = c;
- }
- else
+ if (!c)
{
f->next = fss->free_fifos;
fss->free_fifos = f;
- f = 0;
+ return 0;
}
-done:
- clib_spinlock_unlock (&fss->chunk_lock);
+ f->start_chunk = c;
+ while (c->next)
+ c = c->next;
+ f->end_chunk = c;
+ f->size = data_bytes;
+ f->fs_hdr = fsh;
- if (f)
- {
- f->size = data_bytes;
- f->fs_hdr = fsh;
- }
return f;
}
svm_fifo_chunk_t *c;
fss = fsh_slice_get (fsh, slice_index);
-
- clib_spinlock_lock (&fss->chunk_lock);
c = fsh_try_alloc_chunk (fsh, fss, chunk_size);
- clib_spinlock_unlock (&fss->chunk_lock);
return c;
}
u32 n_collect = 0, fl_index;
svm_fifo_chunk_t *next;
- clib_spinlock_lock (&fss->chunk_lock);
-
while (c)
{
CLIB_MEM_UNPOISON (c, sizeof (*c));
c = next;
}
- fss->n_fl_chunk_bytes += n_collect;
+ fss_fl_chunk_bytes_add (fss, n_collect);
fsh_cached_bytes_add (fsh, n_collect);
-
- clib_spinlock_unlock (&fss->chunk_lock);
}
void
for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
{
fss = fsh_slice_get (fsh, slice_index);
- n_bytes += fss->n_fl_chunk_bytes;
+ n_bytes += fss_fl_chunk_bytes (fss);
}
return n_bytes;