typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- struct rte_mbuf ***mbuf_pending_free_list;
-
- /* cached last pool */
- struct rte_mempool *last_pool;
- u8 last_buffer_pool_index;
+ struct rte_mbuf **mbuf_alloc_list;
} dpdk_buffer_per_thread_data;
typedef struct
dpdk_buffer_main_t dpdk_buffer_main;
static_always_inline void
-dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b)
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b,
+ int maybe_next)
{
- vlib_buffer_t *hb = b;
- dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
- dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
struct rte_mbuf *mb;
u32 next, flags;
- mb = rte_mbuf_from_vlib_buffer (hb);
next:
flags = b->flags;
b->n_add_refs = 0;
}
- mb = rte_pktmbuf_prefree_seg (mb);
- if (mb)
- {
- if (mb->pool != d->last_pool)
- {
- d->last_pool = mb->pool;
- dpdk_mempool_private_t *privp = rte_mempool_get_priv (d->last_pool);
- d->last_buffer_pool_index = privp->buffer_pool_index;
- vec_validate_aligned (d->mbuf_pending_free_list,
- d->last_buffer_pool_index,
- CLIB_CACHE_LINE_BYTES);
- }
- vec_add1 (d->mbuf_pending_free_list[d->last_buffer_pool_index], mb);
- }
+ if ((mb = rte_pktmbuf_prefree_seg (mb)))
+ rte_mempool_put (mb->pool, mb);
- if (flags & VLIB_BUFFER_NEXT_PRESENT)
+ if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
{
b = vlib_get_buffer (vm, next);
goto next;
for (i = 0; i < vec_len (f->buffers); i++)
{
b = vlib_get_buffer (vm, f->buffers[i]);
- dpdk_rte_pktmbuf_free (vm, thread_index, b);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
}
vec_free (f->name);
uword min_free_buffers)
{
dpdk_main_t *dm = &dpdk_main;
- vlib_buffer_t *b0, *b1, *b2, *b3;
- int n, i;
- u32 bi0, bi1, bi2, bi3;
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
+ struct rte_mbuf **mb;
+ uword n_left, first;
+ word n_alloc;
unsigned socket_id = rte_socket_id ();
+ u32 thread_index = vlib_get_thread_index ();
+ dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
- struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
vlib_buffer_t bt;
+ u32 *bi;
/* Too early? */
if (PREDICT_FALSE (rmp == 0))
return 0;
/* Already have enough free buffers on free list? */
- n = min_free_buffers - vec_len (fl->buffers);
- if (n <= 0)
+ n_alloc = min_free_buffers - vec_len (fl->buffers);
+ if (n_alloc <= 0)
return min_free_buffers;
/* Always allocate round number of buffers. */
- n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
+ n_alloc = round_pow2 (n_alloc, CLIB_CACHE_LINE_BYTES / sizeof (u32));
/* Always allocate new buffers in reasonably large sized chunks. */
- n = clib_max (n, fl->min_n_buffers_each_alloc);
+ n_alloc = clib_max (n_alloc, fl->min_n_buffers_each_alloc);
- vec_validate_aligned (vm->mbuf_alloc_list, n - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (d->mbuf_alloc_list, n_alloc - 1,
+ CLIB_CACHE_LINE_BYTES);
- if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0)
+ if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0)
return 0;
memset (&bt, 0, sizeof (vlib_buffer_t));
vlib_buffer_init_for_free_list (&bt, fl);
bt.buffer_pool_index = privp->buffer_pool_index;
- _vec_len (vm->mbuf_alloc_list) = n;
+ _vec_len (d->mbuf_alloc_list) = n_alloc;
- i = 0;
- int f = vec_len (fl->buffers);
- vec_resize_aligned (fl->buffers, n, CLIB_CACHE_LINE_BYTES);
+ first = vec_len (fl->buffers);
+ vec_resize_aligned (fl->buffers, n_alloc, CLIB_CACHE_LINE_BYTES);
- while (i < (n - 7))
- {
- vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
- (vm->mbuf_alloc_list[i + 4]), STORE);
- vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
- (vm->mbuf_alloc_list[i + 5]), STORE);
- vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
- (vm->mbuf_alloc_list[i + 6]), STORE);
- vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
- (vm->mbuf_alloc_list[i + 7]), STORE);
-
- mb0 = vm->mbuf_alloc_list[i];
- mb1 = vm->mbuf_alloc_list[i + 1];
- mb2 = vm->mbuf_alloc_list[i + 2];
- mb3 = vm->mbuf_alloc_list[i + 3];
-
- b0 = vlib_buffer_from_rte_mbuf (mb0);
- b1 = vlib_buffer_from_rte_mbuf (mb1);
- b2 = vlib_buffer_from_rte_mbuf (mb2);
- b3 = vlib_buffer_from_rte_mbuf (mb3);
-
- bi0 = vlib_get_buffer_index (vm, b0);
- bi1 = vlib_get_buffer_index (vm, b1);
- bi2 = vlib_get_buffer_index (vm, b2);
- bi3 = vlib_get_buffer_index (vm, b3);
-
- fl->buffers[f++] = bi0;
- fl->buffers[f++] = bi1;
- fl->buffers[f++] = bi2;
- fl->buffers[f++] = bi3;
-
- clib_memcpy64_x4 (b0, b1, b2, b3, &bt);
-
- if (fl->buffer_init_function)
- {
- fl->buffer_init_function (vm, fl, &bi0, 1);
- fl->buffer_init_function (vm, fl, &bi1, 1);
- fl->buffer_init_function (vm, fl, &bi2, 1);
- fl->buffer_init_function (vm, fl, &bi3, 1);
- }
- i += 4;
- }
-
- while (i < n)
- {
- mb0 = vm->mbuf_alloc_list[i];
+ n_left = n_alloc;
+ mb = d->mbuf_alloc_list;
+ bi = fl->buffers + first;
- b0 = vlib_buffer_from_rte_mbuf (mb0);
- bi0 = vlib_get_buffer_index (vm, b0);
+ ASSERT (n_left % 8 == 0);
- fl->buffers[f++] = bi0;
- clib_memcpy (b0, &bt, sizeof (vlib_buffer_t));
-
- if (fl->buffer_init_function)
- fl->buffer_init_function (vm, fl, &bi0, 1);
- i++;
+ while (n_left >= 8)
+ {
+ if (PREDICT_FALSE (n_left < 24))
+ goto no_prefetch;
+
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[16]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[17]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[18]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[19]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[20]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[21]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[22]), STORE);
+ vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[23]), STORE);
+
+ no_prefetch:
+ vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8,
+ sizeof (struct rte_mbuf));
+ clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[0]),
+ vlib_buffer_from_rte_mbuf (mb[1]),
+ vlib_buffer_from_rte_mbuf (mb[2]),
+ vlib_buffer_from_rte_mbuf (mb[3]), &bt);
+ clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[4]),
+ vlib_buffer_from_rte_mbuf (mb[5]),
+ vlib_buffer_from_rte_mbuf (mb[6]),
+ vlib_buffer_from_rte_mbuf (mb[7]), &bt);
+
+ n_left -= 8;
+ mb += 8;
+ bi += 8;
}
- fl->n_alloc += n;
+ if (fl->buffer_init_function)
+ fl->buffer_init_function (vm, fl, fl->buffers + first, n_alloc);
+
+ fl->n_alloc += n_alloc;
- return n;
+ return n_alloc;
}
static_always_inline void
-dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
+dpdk_prefetch_buffer (vlib_buffer_t * b)
{
- vlib_buffer_t *b;
struct rte_mbuf *mb;
- b = vlib_get_buffer (vm, bi);
mb = rte_mbuf_from_vlib_buffer (b);
- CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+ CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
}
already_announced:
;
}
- else
- {
- if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
- dpdk_rte_pktmbuf_free (vm, thread_index, b);
- }
+ else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+ dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
}
static_always_inline void
u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
{
vlib_buffer_main_t *bm = &buffer_main;
- dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
- vlib_buffer_t *b0, *b1, *b2, *b3;
+ vlib_buffer_t *bufp[n_buffers], **b = bufp;
u32 thread_index = vlib_get_thread_index ();
- dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
int i = 0;
+ u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE |
+ VLIB_BUFFER_NEXT_PRESENT);
+ u32 n_left, *bi;
u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
u32 follow_buffer_next);
if (!n_buffers)
return;
- while (i + 7 < n_buffers)
+ n_left = n_buffers;
+ bi = buffers;
+ b = bufp;
+ vlib_get_buffers (vm, bi, b, n_buffers);
+
+ while (n_left >= 4)
{
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 4]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 5]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 6]);
- dpdk_prefetch_buffer_by_index (vm, buffers[i + 7]);
-
- b0 = vlib_get_buffer (vm, buffers[i]);
- b1 = vlib_get_buffer (vm, buffers[i + 1]);
- b2 = vlib_get_buffer (vm, buffers[i + 2]);
- b3 = vlib_get_buffer (vm, buffers[i + 3]);
-
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
-
- recycle_or_free (vm, bm, buffers[i], b0);
- recycle_or_free (vm, bm, buffers[i + 1], b1);
- recycle_or_free (vm, bm, buffers[i + 2], b2);
- recycle_or_free (vm, bm, buffers[i + 3], b3);
-
- i += 4;
+ u32 or_flags;
+ vlib_buffer_t **p;
+
+ if (n_left < 16)
+ goto no_prefetch;
+
+ p = b + 12;
+ dpdk_prefetch_buffer (p[0]);
+ dpdk_prefetch_buffer (p[1]);
+ dpdk_prefetch_buffer (p[2]);
+ dpdk_prefetch_buffer (p[3]);
+ no_prefetch:
+
+ for (i = 0; i < 4; i++)
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[i]);
+
+ or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
+
+ if (or_flags & simple_mask)
+ {
+ recycle_or_free (vm, bm, bi[0], b[0]);
+ recycle_or_free (vm, bm, bi[1], b[1]);
+ recycle_or_free (vm, bm, bi[2], b[2]);
+ recycle_or_free (vm, bm, bi[3], b[3]);
+ }
+ else
+ {
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0);
+ dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0);
+ }
+ bi += 4;
+ b += 4;
+ n_left -= 4;
}
- while (i < n_buffers)
+ while (n_left)
{
- b0 = vlib_get_buffer (vm, buffers[i]);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- recycle_or_free (vm, bm, buffers[i], b0);
- i++;
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
+ recycle_or_free (vm, bm, bi[0], b[0]);
+ bi += 1;
+ b += 1;
+ n_left -= 1;
}
if (vec_len (vm->buffer_announce_list))
{
}
_vec_len (vm->buffer_announce_list) = 0;
}
-
- vec_foreach_index (i, d->mbuf_pending_free_list)
- {
- int len = vec_len (d->mbuf_pending_free_list[i]);
- if (len)
- {
- rte_mempool_put_bulk (d->mbuf_pending_free_list[i][len - 1]->pool,
- (void *) d->mbuf_pending_free_list[i], len);
- vec_reset_length (d->mbuf_pending_free_list[i]);
- }
- }
}
void
clib_error_t *error = 0;
u32 size, obj_size;
i32 ret;
+ uword i;
obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0);
+
+#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0);
+#else
+ size = rte_mempool_calc_mem_size_helper (num_elts, obj_size, 21);
+#endif
- error =
- vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa,
- VLIB_PHYSMEM_F_HUGETLB | VLIB_PHYSMEM_F_SHARED,
- pri);
+ error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa,
+ VLIB_PHYSMEM_F_HUGETLB |
+ VLIB_PHYSMEM_F_SHARED, pri);
if (error)
return error;
pr = vlib_physmem_get_region (vm, pri[0]);
- mp =
- rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size,
- 512, pool_priv_size, numa, 0);
+ mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size,
+ 512, pool_priv_size, numa, 0);
if (!mp)
return clib_error_return (0, "failed to create %s", pool_name);
priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE;
rte_pktmbuf_pool_init (mp, &priv);
- ret =
- rte_mempool_populate_iova_tab (mp, pr->mem, pr->page_table, pr->n_pages,
- pr->log2_page_size, NULL, NULL);
- if (ret != (i32) mp->size)
+ for (i = 0; i < pr->n_pages; i++)
{
- rte_mempool_free (mp);
- return clib_error_return (0, "failed to populate %s", pool_name);
+ size_t page_size = 1 << pr->log2_page_size;
+ ret = rte_mempool_populate_iova (mp, ((char *) pr->mem) + i * page_size,
+ pr->page_table[i], page_size, 0, 0);
+ if (ret < 0)
+ {
+ rte_mempool_free (mp);
+ return clib_error_return (0, "failed to populate %s", pool_name);
+ }
}
_mp[0] = mp;
{
void *oldheap;
- vlib_buffer_state_heap = mheap_alloc (0, 10 << 20);
-
+ vlib_buffer_state_heap =
+ mheap_alloc_with_lock (0, 10 << 20, 0 /* locked */ );
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword));
Code Review / vpp.git / blobdiff