X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fbuffer.c;h=2379a9aa7d0dd71e35f39ed973019f0d5b85b5c2;hb=efe875e7cd4bbdf0d55374c7a49577f02a92a3fd;hp=6add0630d8509d843c6ac3c5468519fad427df3a;hpb=a8c0b62a88494e9a8562c57dfd3fd75818a629a7;p=vpp.git diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 6add0630d85..2379a9aa7d0 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -19,7 +19,9 @@ #include #include #include +#include #include +#include #include #include @@ -27,9 +29,11 @@ STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM, "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM"); +extern struct rte_mbuf *dpdk_mbuf_template_by_pool_index; #ifndef CLIB_MARCH_VARIANT struct rte_mempool **dpdk_mempool_by_buffer_pool_index = 0; struct rte_mempool **dpdk_no_cache_mempool_by_buffer_pool_index = 0; +struct rte_mbuf *dpdk_mbuf_template_by_pool_index = 0; clib_error_t * dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) @@ -38,7 +42,7 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) struct rte_mempool *mp, *nmp; struct rte_pktmbuf_pool_private priv; enum rte_iova_mode iova_mode; - u32 *bi; + u32 i; u8 *name = 0; u32 elt_size = @@ -52,7 +56,7 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) /* normal mempool */ name = format (name, "vpp pool %u%c", bp->index, 0); - mp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers), + mp = rte_mempool_create_empty ((char *) name, bp->n_buffers, elt_size, 512, sizeof (priv), bp->numa_node, 0); if (!mp) @@ -66,7 +70,7 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) /* non-cached mempool */ name = format (name, "vpp pool %u (no cache)%c", bp->index, 0); - nmp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers), + nmp = rte_mempool_create_empty ((char *) name, bp->n_buffers, elt_size, 0, sizeof (priv), bp->numa_node, 0); if (!nmp) @@ -88,6 +92,7 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) rte_mempool_set_ops_byname (nmp, "vpp-no-cache", NULL); /* Call the mempool priv initializer */ + memset (&priv, 0, sizeof (priv)); priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + vlib_buffer_get_default_data_size (vm); priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; @@ -97,11 +102,10 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) iova_mode = rte_eal_iova_mode (); /* populate mempool object buffer header */ - /* *INDENT-OFF* */ - vec_foreach (bi, bp->buffers) + for (i = 0; i < bp->n_buffers; i++) { struct rte_mempool_objhdr *hdr; - vlib_buffer_t *b = vlib_get_buffer (vm, *bi); + vlib_buffer_t *b = vlib_get_buffer (vm, bp->buffers[i]); struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer (b); hdr = (struct rte_mempool_objhdr *) RTE_PTR_SUB (mb, sizeof (*hdr)); hdr->mp = mp; @@ -112,22 +116,30 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) mp->populated_size++; nmp->populated_size++; } - /* *INDENT-ON* */ +#if RTE_VERSION >= RTE_VERSION_NUM(22, 3, 0, 0) + mp->flags &= ~RTE_MEMPOOL_F_NON_IO; +#endif /* call the object initializers */ rte_mempool_obj_iter (mp, rte_pktmbuf_init, 0); - /* *INDENT-OFF* */ - vec_foreach (bi, bp->buffers) + /* create mbuf header tempate from the first buffer in the pool */ + vec_validate_aligned (dpdk_mbuf_template_by_pool_index, bp->index, + CLIB_CACHE_LINE_BYTES); + clib_memcpy (vec_elt_at_index (dpdk_mbuf_template_by_pool_index, bp->index), + rte_mbuf_from_vlib_buffer (vlib_buffer_ptr_from_index + (buffer_mem_start, *bp->buffers, + 0)), sizeof (struct rte_mbuf)); + + for (i = 0; i < bp->n_buffers; i++) { vlib_buffer_t *b; - b = vlib_buffer_ptr_from_index (buffer_mem_start, *bi, 0); - vlib_buffer_copy_template (b, &bp->buffer_template); + b = vlib_buffer_ptr_from_index (buffer_mem_start, bp->buffers[i], 0); + b->template = bp->buffer_template; } - /* *INDENT-ON* */ /* map DMA pages if at least one physical device exists */ - if (rte_eth_dev_count_avail ()) + if (rte_eth_dev_count_avail () || rte_cryptodev_count ()) { uword i; size_t page_sz; @@ -144,7 +156,12 @@ dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) pointer_to_uword (va) : pm->page_table[i]; if (do_vfio_map && +#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0) rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz)) +#else + rte_vfio_container_dma_map (RTE_VFIO_DEFAULT_CONTAINER_FD, + pointer_to_uword (va), pa, page_sz)) +#endif do_vfio_map = 0; struct rte_mempool_memhdr *memhdr; @@ -180,7 +197,7 @@ dpdk_ops_vpp_free (struct rte_mempool *mp) #endif static_always_inline void -dpdk_ops_vpp_enqueue_one (vlib_buffer_t * bt, void *obj) +dpdk_ops_vpp_enqueue_one (vlib_buffer_template_t *bt, void *obj) { /* Only non-replicated packets (b->ref_count == 1) expected */ @@ -188,7 +205,7 @@ dpdk_ops_vpp_enqueue_one (vlib_buffer_t * bt, void *obj) vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); ASSERT (b->ref_count == 1); ASSERT (b->buffer_pool_index == bt->buffer_pool_index); - vlib_buffer_copy_template (b, bt); + b->template = *bt; } int @@ -197,14 +214,14 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue) (struct rte_mempool * mp, { const int batch_size = 32; vlib_main_t *vm = vlib_get_main (); - vlib_buffer_t bt; + vlib_buffer_template_t bt; u8 buffer_pool_index = mp->pool_id; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); u32 bufs[batch_size]; u32 n_left = n; void *const *obj = obj_table; - vlib_buffer_copy_template (&bt, &bp->buffer_template); + bt = bp->buffer_template; while (n_left >= 4) { @@ -246,9 +263,9 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue) (struct rte_mempool * mp, CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue); static_always_inline void -dpdk_ops_vpp_enqueue_no_cache_one (vlib_main_t * vm, struct rte_mempool *old, +dpdk_ops_vpp_enqueue_no_cache_one (vlib_main_t *vm, struct rte_mempool *old, struct rte_mempool *new, void *obj, - vlib_buffer_t * bt) + vlib_buffer_template_t *bt) { struct rte_mbuf *mb = obj; vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); @@ -256,8 +273,7 @@ dpdk_ops_vpp_enqueue_no_cache_one (vlib_main_t * vm, struct rte_mempool *old, if (clib_atomic_sub_fetch (&b->ref_count, 1) == 0) { u32 bi = vlib_get_buffer_index (vm, b); - mb->pool = new; - vlib_buffer_copy_template (b, bt); + b->template = *bt; vlib_buffer_pool_put (vm, bt->buffer_pool_index, &bi, 1); return; } @@ -269,12 +285,12 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue_no_cache) (struct rte_mempool * cmp, unsigned n) { vlib_main_t *vm = vlib_get_main (); - vlib_buffer_t bt; + vlib_buffer_template_t bt; struct rte_mempool *mp; mp = dpdk_mempool_by_buffer_pool_index[cmp->pool_id]; u8 buffer_pool_index = cmp->pool_id; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); - vlib_buffer_copy_template (&bt, &bp->buffer_template); + bt = bp->buffer_template; while (n >= 4) { @@ -298,6 +314,37 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue_no_cache) (struct rte_mempool * cmp, CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue_no_cache); +static_always_inline void +dpdk_mbuf_init_from_template (struct rte_mbuf **mba, struct rte_mbuf *mt, + int count) +{ + /* Assumptions about rte_mbuf layout */ + STATIC_ASSERT_OFFSET_OF (struct rte_mbuf, buf_addr, 0); + STATIC_ASSERT_OFFSET_OF (struct rte_mbuf, buf_iova, 8); + STATIC_ASSERT_SIZEOF_ELT (struct rte_mbuf, buf_iova, 8); + STATIC_ASSERT_SIZEOF_ELT (struct rte_mbuf, buf_iova, 8); + STATIC_ASSERT_SIZEOF (struct rte_mbuf, 128); + + while (count--) + { + struct rte_mbuf *mb = mba[0]; + int i; + /* bytes 0 .. 15 hold buf_addr and buf_iova which we need to preserve */ + /* copy bytes 16 .. 31 */ + *((u8x16 *) mb + 1) = *((u8x16 *) mt + 1); + + /* copy bytes 32 .. 127 */ +#ifdef CLIB_HAVE_VEC256 + for (i = 1; i < 4; i++) + *((u8x32 *) mb + i) = *((u8x32 *) mt + i); +#else + for (i = 2; i < 8; i++) + *((u8x16 *) mb + i) = *((u8x16 *) mt + i); +#endif + mba++; + } +} + int CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp, void **obj_table, unsigned n) @@ -307,6 +354,7 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp, u32 bufs[batch_size], total = 0, n_alloc = 0; u8 buffer_pool_index = mp->pool_id; void **obj = obj_table; + struct rte_mbuf t = dpdk_mbuf_template_by_pool_index[buffer_pool_index]; while (n >= batch_size) { @@ -317,6 +365,7 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp, vlib_get_buffers_with_offset (vm, bufs, obj, batch_size, -(i32) sizeof (struct rte_mbuf)); + dpdk_mbuf_init_from_template ((struct rte_mbuf **) obj, &t, batch_size); total += batch_size; obj += batch_size; n -= batch_size; @@ -331,6 +380,7 @@ CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp, vlib_get_buffers_with_offset (vm, bufs, obj, n, -(i32) sizeof (struct rte_mbuf)); + dpdk_mbuf_init_from_template ((struct rte_mbuf **) obj, &t, n); } return 0; @@ -368,7 +418,15 @@ dpdk_ops_vpp_dequeue_no_cache (struct rte_mempool *mp, void **obj_table, static unsigned dpdk_ops_vpp_get_count (const struct rte_mempool *mp) { - clib_warning (""); + vlib_main_t *vm = vlib_get_main (); + if (mp) + { + vlib_buffer_pool_t *pool = vlib_get_buffer_pool (vm, mp->pool_id); + if (pool) + { + return pool->n_avail; + } + } return 0; }