return c;
}
+static void
+fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
+ svm_fifo_shared_t *sf)
+{
+ sf->next = fss->free_fifos;
+ fss->free_fifos = fs_sptr (fsh, sf);
+}
+
+static void
+fss_fifo_free_list_push_list (fifo_segment_header_t *fsh,
+ fifo_segment_slice_t *fss,
+ svm_fifo_shared_t *head, svm_fifo_shared_t *tail)
+{
+ tail->next = fss->free_fifos;
+ fss->free_fifos = fs_sptr (fsh, head);
+}
+
+svm_fifo_shared_t *
+fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
+{
+ svm_fifo_shared_t *sf;
+ sf = fs_ptr (fsh, fss->free_fifos);
+ fss->free_fifos = sf->next;
+ return sf;
+}
+
static inline void
pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f)
{
fsh->max_log2_fifo_size = min_log2 (max_fifo);
fsh->n_cached_bytes = 0;
fsh->n_reserved_bytes = fsh->byte_index;
+ fsh->start_byte_index = fsh->byte_index;
ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset);
fs->max_byte_index = fsh->max_byte_index;
fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
fifo_segment_slice_t * fss, u32 batch_size)
{
- svm_fifo_shared_t *f;
+ svm_fifo_shared_t *f, *head = 0, *tail;
uword size;
u8 *fmem;
int i;
return -1;
/* Carve fifo hdr space */
+ tail = f = (svm_fifo_shared_t *) fmem;
for (i = 0; i < batch_size; i++)
{
- f = (svm_fifo_shared_t *) fmem;
- memset (f, 0, sizeof (*f));
- f->next = fss->free_fifos;
- fss->free_fifos = f;
+ clib_memset (f, 0, sizeof (*f));
+ f->next = fs_sptr (fsh, head);
+ head = f;
fmem += sizeof (*f);
+ f = (svm_fifo_shared_t *) fmem;
}
+ fss_fifo_free_list_push_list (fsh, fss, head, tail);
+
return 0;
}
static svm_fifo_shared_t *
fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
{
- svm_fifo_shared_t *f;
+ svm_fifo_shared_t *sf;
if (!fss->free_fifos)
{
return 0;
}
- f = fss->free_fifos;
- fss->free_fifos = f->next;
- memset (f, 0, sizeof (*f));
- return f;
+ sf = fss_fifo_free_list_pop (fsh, fss);
+ clib_memset (sf, 0, sizeof (*sf));
+
+ return sf;
}
static svm_fifo_chunk_t *
c = fsh_try_alloc_chunk (fsh, fss, min_size);
if (!c)
{
- sf->next = fss->free_fifos;
- fss->free_fifos = sf;
+ fss_fifo_free_list_push (fsh, fss, sf);
return 0;
}
sf->head_chunk = sf->tail_chunk = 0;
/* Add to free list */
- sf->next = fss->free_fifos;
- fss->free_fifos = sf;
+ fss_fifo_free_list_push (fsh, fss, sf);
fss->virtual_mem -= svm_fifo_size (f);
}
void
-fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f)
+fifo_segment_detach_fifo (fifo_segment_t *fs, svm_fifo_t **f)
{
fifo_slice_private_t *pfss;
fifo_segment_slice_t *fss;
- svm_fifo_chunk_t *c;
- u32 fl_index;
-
- ASSERT (f->refcnt == 1);
+ u32 fl_index, slice_index;
+ svm_fifo_chunk_t **c;
+ svm_fifo_t *of = *f;
- fss = fsh_slice_get (fs->h, f->shr->slice_index);
- pfss = fs_slice_private_get (fs, f->shr->slice_index);
- fss->virtual_mem -= svm_fifo_size (f);
- if (f->flags & SVM_FIFO_F_LL_TRACKED)
- pfss_fifo_del_active_list (pfss, f);
+ slice_index = of->master_thread_index;
+ fss = fsh_slice_get (fs->h, slice_index);
+ pfss = fs_slice_private_get (fs, slice_index);
+ fss->virtual_mem -= svm_fifo_size (of);
+ if (of->flags & SVM_FIFO_F_LL_TRACKED)
+ pfss_fifo_del_active_list (pfss, of);
- c = fs_chunk_ptr (fs->h, f->shr->start_chunk);
- while (c)
+ /* Update slice counts for chunks that were detached */
+ vec_foreach (c, of->chunks_at_attach)
{
- fl_index = fs_freelist_for_size (c->length);
+ fl_index = fs_freelist_for_size ((*c)->length);
clib_atomic_fetch_sub_rel (&fss->num_chunks[fl_index], 1);
- c = fs_chunk_ptr (fs->h, c->next);
}
+ vec_free (of->chunks_at_attach);
+
+ clib_mem_bulk_free (pfss->fifos, *f);
+ *f = 0;
}
void
-fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f,
- u32 slice_index)
+fifo_segment_attach_fifo (fifo_segment_t *fs, svm_fifo_t **f, u32 slice_index)
{
fifo_slice_private_t *pfss;
fifo_segment_slice_t *fss;
svm_fifo_chunk_t *c;
+ svm_fifo_t *nf, *of;
u32 fl_index;
- f->shr->slice_index = slice_index;
- fss = fsh_slice_get (fs->h, f->shr->slice_index);
- pfss = fs_slice_private_get (fs, f->shr->slice_index);
- fss->virtual_mem += svm_fifo_size (f);
- if (f->flags & SVM_FIFO_F_LL_TRACKED)
- pfss_fifo_add_active_list (pfss, f);
+ nf = fs_fifo_alloc (fs, slice_index);
+ clib_memcpy_fast (nf, *f, sizeof (*nf));
+
+ fss = fsh_slice_get (fs->h, slice_index);
+ pfss = fs_slice_private_get (fs, slice_index);
+ fss->virtual_mem += svm_fifo_size (nf);
+ if (nf->flags & SVM_FIFO_F_LL_TRACKED)
+ pfss_fifo_add_active_list (pfss, nf);
+
+ /* Update allocated chunks for fifo segment and build list
+ * of chunks to be freed at detach */
+ of = *f;
+ of->chunks_at_attach = 0;
- c = fs_chunk_ptr (fs->h, f->shr->start_chunk);
+ c = fs_chunk_ptr (fs->h, nf->shr->start_chunk);
while (c)
{
fl_index = fs_freelist_for_size (c->length);
clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1);
+ vec_add1 (of->chunks_at_attach, c);
c = fs_chunk_ptr (fs->h, c->next);
}
+
+ nf->shr->slice_index = slice_index;
+ *f = nf;
}
uword
mq = vec_elt_at_index (fs->mqs, mq_index);
- if (!mq->q)
+ if (!mq->q.shr)
{
svm_msg_q_shared_t *smq;
smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset);
return mq;
}
+void
+fifo_segment_msg_qs_discover (fifo_segment_t *fs, int *fds, u32 n_fds)
+{
+ svm_msg_q_shared_t *smq;
+ u32 n_mqs, size, i;
+ uword offset = 0, n_alloced;
+ svm_msg_q_t *mq;
+
+ n_mqs = fs->h->n_mqs;
+ if (n_fds && n_mqs != n_fds)
+ {
+ clib_warning ("expected %u fds got %u", n_mqs, n_fds);
+ return;
+ }
+
+ vec_validate (fs->mqs, n_mqs - 1);
+ n_alloced = fs->h->n_reserved_bytes - fs->h->start_byte_index;
+ ASSERT (n_alloced % n_mqs == 0);
+ size = n_alloced / n_mqs;
+
+ offset = fs->h->start_byte_index;
+ for (i = 0; i < n_mqs; i++)
+ {
+ mq = vec_elt_at_index (fs->mqs, i);
+ smq = (svm_msg_q_shared_t *) ((u8 *) fs->h + offset);
+ svm_msg_q_attach (mq, smq);
+ if (n_fds)
+ svm_msg_q_set_eventfd (mq, fds[i]);
+ offset += size;
+ }
+}
+
uword
fifo_segment_msg_q_offset (fifo_segment_t *fs, u32 mq_index)
{
svm_msg_q_t *mq = vec_elt_at_index (fs->mqs, mq_index);
- if (mq->q == 0)
+ if (mq->q.shr == 0)
return ~0ULL;
- return (uword) ((u8 *) mq->q - (u8 *) fs->h) - sizeof (svm_msg_q_shared_t);
+ return (uword) ((u8 *) mq->q.shr - (u8 *) fs->h) -
+ sizeof (svm_msg_q_shared_t);
}
int
}
static u32
-fs_slice_num_free_fifos (fifo_segment_slice_t * fss)
+fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
{
svm_fifo_shared_t *f;
u32 count = 0;
- f = fss->free_fifos;
+ f = fs_ptr (fsh, fss->free_fifos);
if (f == 0)
return 0;
while (f)
{
- f = f->next;
+ f = fs_ptr (fsh, f->next);
count++;
}
return count;
for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
{
fss = fsh_slice_get (fsh, slice_index);
- count += fs_slice_num_free_fifos (fss);
+ count += fs_slice_num_free_fifos (fsh, fss);
}
return count;
}
Code Review / vpp.git / blobdiff