}
static inline uword
-fsh_virtual_mem (fifo_segment_header_t * fsh)
+fs_virtual_mem (fifo_segment_t *fs)
{
+ fifo_segment_header_t *fsh = fs->h;
fifo_segment_slice_t *fss;
uword total_vm = 0;
int i;
- for (i = 0; i < fsh->n_slices; i++)
+ for (i = 0; i < fs->n_slices; i++)
{
fss = fsh_slice_get (fsh, i);
total_vm += clib_atomic_load_relax_n (&fss->virtual_mem);
csp = fs_chunk_sptr (fsh, c);
ASSERT (csp <= FS_CL_HEAD_MASK);
- old_head = clib_atomic_load_relax_n (&fss->free_chunks[fl_index]);
+ old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
do
{
c->next = old_head & FS_CL_HEAD_MASK;
new_head = csp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
}
- while (!clib_atomic_cmp_and_swap_acq_relax (
- &fss->free_chunks[fl_index], &old_head, &new_head, 1 /* weak */));
+ while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
+ &new_head, 0 /* weak */, __ATOMIC_RELEASE,
+ __ATOMIC_ACQUIRE));
}
static void
headsp = fs_chunk_sptr (fsh, head);
ASSERT (headsp <= FS_CL_HEAD_MASK);
- old_head = clib_atomic_load_relax_n (&fss->free_chunks[fl_index]);
+ old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
do
{
tail->next = old_head & FS_CL_HEAD_MASK;
new_head = headsp + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
}
- while (!clib_atomic_cmp_and_swap_acq_relax (
- &fss->free_chunks[fl_index], &old_head, &new_head, 1 /* weak */));
+ while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
+ &new_head, 0 /* weak */, __ATOMIC_RELEASE,
+ __ATOMIC_ACQUIRE));
}
static svm_fifo_chunk_t *
ASSERT (fss_chunk_fl_index_is_valid (fss, fl_index));
- old_head = clib_atomic_load_relax_n (&fss->free_chunks[fl_index]);
+ old_head = clib_atomic_load_acq_n (&fss->free_chunks[fl_index]);
/* Lock-free stacks are affected by ABA if a side allocates a chunk and
* shortly thereafter frees it. To circumvent that, reuse the upper bits
c = fs_chunk_ptr (fsh, old_head & FS_CL_HEAD_MASK);
new_head = c->next + ((old_head + FS_CL_HEAD_TINC) & FS_CL_HEAD_TMASK);
}
- while (!clib_atomic_cmp_and_swap_acq_relax (
- &fss->free_chunks[fl_index], &old_head, &new_head, 1 /* weak */));
+ while (!__atomic_compare_exchange (&fss->free_chunks[fl_index], &old_head,
+ &new_head, 0 /* weak */, __ATOMIC_RELEASE,
+ __ATOMIC_ACQUIRE));
return c;
}
int
fifo_segment_init (fifo_segment_t * fs)
{
- u32 align = 8, offset = 2 * 4096, slices_sz, i;
+ u32 align = 8, offset = FIFO_SEGMENT_ALLOC_OVERHEAD, slices_sz, i;
uword max_fifo, seg_start, seg_sz;
fifo_segment_header_t *fsh;
ssvm_shared_header_t *sh;
seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align);
fsh = uword_to_pointer (seg_start, void *);
- CLIB_MEM_UNPOISON (fsh, seg_sz);
+ clib_mem_unpoison (fsh, seg_sz);
memset (fsh, 0, sizeof (*fsh) + slices_sz);
fsh->byte_index = sizeof (*fsh) + slices_sz;
pool_get_zero (sm->segments, fs);
+ fs->fs_index = fs - sm->segments;
+ fs->sm_index = ~0;
fs->ssvm.ssvm_size = a->segment_size;
fs->ssvm.my_pid = getpid ();
fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
if (data_bytes <= fss_fl_chunk_bytes (fss) + n_free)
{
u32 min_size = FIFO_SEGMENT_MIN_FIFO_SIZE;
-
+ if (n_free < min_size)
+ goto done;
batch = (data_bytes - fss_fl_chunk_bytes (fss)) / min_size;
batch = clib_min (batch + 1, n_free / min_size);
if (fsh_try_alloc_chunk_batch (fsh, fss, 0, batch))
while (c)
{
- CLIB_MEM_UNPOISON (c, sizeof (*c));
+ clib_mem_unpoison (c, sizeof (*c));
next = fs_chunk_ptr (fsh, c->next);
fl_index = fs_freelist_for_size (c->length);
fss_chunk_free_list_push (fsh, fss, fl_index, c);
vec_free (fs->slices);
- vec_foreach (fs->mqs, mq)
+ vec_foreach (mq, fs->mqs)
svm_msg_q_cleanup (mq);
vec_free (fs->mqs);
svm_fifo_init (f, data_bytes);
+ f->segment_manager = fs->sm_index;
+ f->segment_index = fs->fs_index;
+
fss = fsh_slice_get (fsh, slice_index);
pfss = fs_slice_private_get (fs, slice_index);
return f;
}
+svm_fifo_t *
+fifo_segment_duplicate_fifo (fifo_segment_t *fs, svm_fifo_t *f)
+{
+ svm_fifo_t *nf = fs_fifo_alloc (fs, 0);
+ clib_memcpy (nf, f, sizeof (*f));
+ return nf;
+}
+
/**
* Free fifo allocated in fifo segment
*/
return (u8 *) f->shr - (u8 *) f->fs_hdr;
}
+svm_fifo_chunk_t *
+fifo_segment_alloc_chunk_w_slice (fifo_segment_t *fs, u32 slice_index,
+ u32 chunk_size)
+{
+ fifo_segment_header_t *fsh = fs->h;
+ fifo_segment_slice_t *fss;
+
+ fss = fsh_slice_get (fsh, slice_index);
+ return fsh_try_alloc_chunk (fsh, fss, chunk_size);
+}
+
+void
+fifo_segment_collect_chunk (fifo_segment_t *fs, u32 slice_index,
+ svm_fifo_chunk_t *c)
+{
+ fsh_collect_chunks (fs->h, slice_index, c);
+}
+
+uword
+fifo_segment_chunk_offset (fifo_segment_t *fs, svm_fifo_chunk_t *c)
+{
+ return (u8 *) c - (u8 *) fs->h;
+}
+
svm_msg_q_t *
fifo_segment_msg_q_alloc (fifo_segment_t *fs, u32 mq_index,
svm_msg_q_cfg_t *cfg)
size = svm_msg_q_size_to_alloc (cfg);
base = fsh_alloc_aligned (fsh, size, 8);
+ if (!base)
+ return 0;
+
fsh->n_reserved_bytes += size;
smq = svm_msg_q_init (base, cfg);
return fs->h->max_byte_index - fs->h->n_reserved_bytes;
}
-u8
-fsh_has_reached_mem_limit (fifo_segment_header_t * fsh)
+static u8
+fs_has_reached_mem_limit (fifo_segment_t *fs)
{
- return (fsh->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0;
+ return (fs->flags & FIFO_SEGMENT_F_MEM_LIMIT) ? 1 : 0;
}
-void
-fsh_reset_mem_limit (fifo_segment_header_t * fsh)
+static void
+fs_reset_mem_limit (fifo_segment_t *fs)
{
- fsh->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT;
+ fs->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT;
}
void *
}
fifo_segment_mem_status_t
-fifo_segment_determine_status (fifo_segment_header_t * fsh, u8 usage)
+fifo_segment_determine_status (fifo_segment_t *fs, u8 usage)
{
- if (!fsh->high_watermark || !fsh->low_watermark)
+ if (!fs->high_watermark || !fs->low_watermark)
return MEMORY_PRESSURE_NO_PRESSURE;
/* once the no-memory is detected, the status continues
* until memory usage gets below the high watermark
*/
- if (fsh_has_reached_mem_limit (fsh))
+ if (fs_has_reached_mem_limit (fs))
{
- if (usage >= fsh->high_watermark)
+ if (usage >= fs->high_watermark)
return MEMORY_PRESSURE_NO_MEMORY;
else
- fsh_reset_mem_limit (fsh);
+ fs_reset_mem_limit (fs);
}
- if (usage >= fsh->high_watermark)
+ if (usage >= fs->high_watermark)
return MEMORY_PRESSURE_HIGH_PRESSURE;
- else if (usage >= fsh->low_watermark)
+ else if (usage >= fs->low_watermark)
return MEMORY_PRESSURE_LOW_PRESSURE;
return MEMORY_PRESSURE_NO_PRESSURE;
fifo_segment_mem_status_t
fifo_segment_get_mem_status (fifo_segment_t * fs)
{
- fifo_segment_header_t *fsh = fs->h;
u8 usage = fifo_segment_get_mem_usage (fs);
- return fifo_segment_determine_status (fsh, usage);
+ return fifo_segment_determine_status (fs, usage);
}
u8 *
f64 usage;
fifo_segment_mem_status_t mem_st;
- indent = format_get_indent (s) + 2;
-
- if (fs == 0)
- {
- s = format (s, "%-20s%10s%15s%15s%15s%15s", "Name", "Type",
- "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
- return s;
- }
+ indent = format_get_indent (s);
fifo_segment_info (fs, &address, &size);
active_fifos = fifo_segment_num_fifos (fs);
free_fifos = fifo_segment_num_free_fifos (fs);
- s = format (s, "%-20v%10U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm),
- format_fifo_segment_type, fs, size >> 20ULL, active_fifos,
- free_fifos, address);
+ s = format (s, "%U%v type: %U size: %U active fifos: %u", format_white_space,
+ 2, ssvm_name (&fs->ssvm), format_fifo_segment_type, fs,
+ format_memory_size, size, active_fifos);
if (!verbose)
return s;
free_chunks = fifo_segment_num_free_chunks (fs, ~0);
if (free_chunks)
- s =
- format (s, "\n\n%UFree/Allocated chunks by size:\n", format_white_space,
- indent + 2);
+ s = format (s, "\n\n%UFree/Allocated chunks by size:\n",
+ format_white_space, indent + 2);
else
s = format (s, "\n");
in_use = fifo_segment_size (fs) - est_free_seg_bytes - tracked_cached_bytes;
usage = (100.0 * in_use) / allocated;
mem_st = fifo_segment_get_mem_status (fs);
- virt = fsh_virtual_mem (fsh);
+ virt = fs_virtual_mem (fs);
reserved = fsh->n_reserved_bytes;
s = format (s, "\n%Useg free bytes: %U (%lu) estimated: %U (%lu) reserved:"
Code Review / vpp.git / blobdiff