From 149ba779cff4ffc48d87ca67a2a8d4afd49b05f5 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 12 Oct 2017 13:09:26 +0200 Subject: [PATCH] vlib: add support for multiple buffer pools Change-Id: Icaf7d7ad47284aea7a56e8006b69f45874d64202 Signed-off-by: Damjan Marion --- src/plugins/dpdk/buffer.c | 22 +++++++++++----------- src/plugins/dpdk/device/common.c | 6 ++++++ src/plugins/dpdk/device/dpdk.h | 1 + src/plugins/dpdk/device/dpdk_priv.h | 8 ++++++++ src/plugins/dpdk/device/node.c | 3 +++ src/plugins/ixge/ixge.c | 12 ++++++++---- src/vlib/buffer.c | 36 ++++++++++++++++++++++++------------ src/vlib/buffer.h | 18 ++++++++++++++---- src/vlib/buffer_funcs.h | 5 ++++- src/vlib/linux/physmem.c | 5 ----- src/vlib/physmem.h | 1 - 11 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index a44428a4592..b3dca95227c 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -193,6 +193,8 @@ fill_free_list (vlib_main_t * vm, if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0) return 0; + dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); + _vec_len (vm->mbuf_alloc_list) = n; i = 0; @@ -233,6 +235,11 @@ fill_free_list (vlib_main_t * vm, vlib_buffer_init_for_free_list (b2, fl); vlib_buffer_init_for_free_list (b3, fl); + b0->buffer_pool_index = privp->buffer_pool_index; + b1->buffer_pool_index = privp->buffer_pool_index; + b2->buffer_pool_index = privp->buffer_pool_index; + b3->buffer_pool_index = privp->buffer_pool_index; + if (fl->buffer_init_function) { fl->buffer_init_function (vm, fl, &bi0, 1); @@ -253,6 +260,7 @@ fill_free_list (vlib_main_t * vm, vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES); vlib_buffer_init_for_free_list (b0, fl); + b0->buffer_pool_index = privp->buffer_pool_index; if (fl->buffer_init_function) fl->buffer_init_function (vm, fl, &bi0, 1); @@ -409,13 +417,6 @@ dpdk_packet_template_init (vlib_main_t * vm, vlib_worker_thread_barrier_release (vm); } -typedef struct -{ - /* must be first */ - struct rte_pktmbuf_pool_private mbp_priv; - vlib_physmem_region_index_t region_index; -} dpdk_mempool_private_t; - clib_error_t * dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, unsigned socket_id) @@ -446,9 +447,8 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, size = rte_mempool_xmem_size (num_mbufs, obj_size, 21); clib_error_t *error = 0; - error = - vlib_physmem_region_alloc (vm, (char *) pool_name, size, socket_id, - VLIB_PHYSMEM_F_HAVE_BUFFERS, &pri); + error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, socket_id, + 0, &pri); if (error) clib_error_report (error); @@ -487,7 +487,7 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0); dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - privp->region_index = pri; + privp->buffer_pool_index = vlib_buffer_add_physmem_region (vm, pri); dm->pktmbuf_pools[socket_id] = rmp; diff --git a/src/plugins/dpdk/device/common.c b/src/plugins/dpdk/device/common.c index aedc3f5227d..c65cec6734d 100644 --- a/src/plugins/dpdk/device/common.c +++ b/src/plugins/dpdk/device/common.c @@ -78,8 +78,11 @@ dpdk_device_setup (dpdk_device_t * xd) dpdk_device_error (xd, "rte_eth_tx_queue_setup", rv); } + vec_validate_aligned (xd->buffer_pool_for_queue, xd->rx_q_used - 1, + CLIB_CACHE_LINE_BYTES); for (j = 0; j < xd->rx_q_used; j++) { + dpdk_mempool_private_t *privp; uword tidx = vnet_get_device_input_thread_index (dm->vnet_main, xd->hw_if_index, j); unsigned lcore = vlib_worker_threads[tidx].lcore_id; @@ -95,6 +98,9 @@ dpdk_device_setup (dpdk_device_t * xd) SOCKET_ID_ANY, 0, dm->pktmbuf_pools[socket_id]); + privp = rte_mempool_get_priv (dm->pktmbuf_pools[socket_id]); + xd->buffer_pool_for_queue[j] = privp->buffer_pool_index; + if (rv < 0) dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv); } diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index a0c57ea21b3..1ed3b280385 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -194,6 +194,7 @@ typedef struct u16 rx_q_used; u16 nb_rx_desc; u16 *cpu_socket_id_by_queue; + u8 *buffer_pool_for_queue; struct rte_eth_conf port_conf; struct rte_eth_txconf tx_conf; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index 52b4ca4b303..a95d03cfba4 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -51,6 +51,14 @@ _(proc-type) \ _(file-prefix) \ _(vdev) +typedef struct +{ + /* must be first */ + struct rte_pktmbuf_pool_private mbp_priv; + u8 buffer_pool_index; +} dpdk_mempool_private_t; + + static inline void dpdk_get_xstats (dpdk_device_t * xd) { diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index cf8b9699206..82978216817 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -347,6 +347,9 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, /* Update buffer template */ vnet_buffer (bt)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; bt->error = node->errors[DPDK_ERROR_NONE]; + /* as DPDK is allocating empty buffers from mempool provided before interface + start for each queue, it is safe to store this in the template */ + bt->buffer_pool_index = xd->buffer_pool_for_queue[queue_id]; mb_index = 0; diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c index 5fd6a901e35..f9d01d55ffd 100644 --- a/src/plugins/ixge/ixge.c +++ b/src/plugins/ixge/ixge.c @@ -2528,8 +2528,9 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) u32 i; dq->tx.head_index_write_back = - vlib_physmem_alloc (vm, vm->buffer_main->physmem_region, &error, - CLIB_CACHE_LINE_BYTES); + vlib_physmem_alloc (vm, + vm->buffer_main->buffer_pools[0].physmem_region, + &error, CLIB_CACHE_LINE_BYTES); for (i = 0; i < dq->n_descriptors; i++) dq->descriptors[i].tx = xm->tx_descriptor_template; @@ -2542,7 +2543,9 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) u64 a; a = - vlib_physmem_virtual_to_physical (vm, vm->buffer_main->physmem_region, + vlib_physmem_virtual_to_physical (vm, + vm->buffer_main-> + buffer_pools[0].physmem_region, dq->descriptors); dr->descriptor_address[0] = a & 0xFFFFFFFF; dr->descriptor_address[1] = a >> (u64) 32; @@ -2570,7 +2573,8 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) a = vlib_physmem_virtual_to_physical (vm, - vm->buffer_main->physmem_region, + vm->buffer_main-> + buffer_pools[0].physmem_region, dq->tx.head_index_write_back); dr->tx.head_index_write_back_address[0] = /* enable bit */ 1 | a; dr->tx.head_index_write_back_address[1] = (u64) a >> (u64) 32; diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index d0d8f604528..f00e8854ac1 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -462,7 +462,7 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) u32 i; for (i = 0; i < vec_len (f->buffer_memory_allocated); i++) - vm->os_physmem_free (vm, vm->buffer_main->physmem_region, + vm->os_physmem_free (vm, vm->buffer_main->buffer_pools[0].physmem_region, f->buffer_memory_allocated[i]); vec_free (f->name); vec_free (f->buffer_memory_allocated); @@ -555,8 +555,10 @@ fill_free_list (vlib_main_t * vm, /* drb: removed power-of-2 ASSERT */ buffers = - vm->os_physmem_alloc_aligned (vm, vm->buffer_main->physmem_region, - n_bytes, sizeof (vlib_buffer_t)); + vm->os_physmem_alloc_aligned (vm, + vm->buffer_main-> + buffer_pools[0].physmem_region, n_bytes, + sizeof (vlib_buffer_t)); if (!buffers) return n_alloc; @@ -960,10 +962,15 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, return copied; } -void -vlib_buffer_add_mem_range (vlib_main_t * vm, uword start, uword size) +u8 +vlib_buffer_add_physmem_region (vlib_main_t * vm, + vlib_physmem_region_index_t pri) { vlib_buffer_main_t *bm = vm->buffer_main; + vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, pri); + vlib_buffer_pool_t *p; + uword start = pointer_to_uword (pr->mem); + uword size = pr->size; if (bm->buffer_mem_size == 0) { @@ -989,6 +996,12 @@ vlib_buffer_add_mem_range (vlib_main_t * vm, uword start, uword size) { clib_panic ("buffer memory size out of range!"); } + + vec_add2 (bm->buffer_pools, p, 1); + p->start = start; + p->size = size; + p->physmem_region = pri; + return p - bm->buffer_pools; } static u8 * @@ -1057,6 +1070,7 @@ clib_error_t * vlib_buffer_main_init (struct vlib_main_t * vm) { vlib_buffer_main_t *bm; + vlib_physmem_region_index_t pri; clib_error_t *error; vec_validate (vm->buffer_main, 0); @@ -1085,12 +1099,10 @@ vlib_buffer_main_init (struct vlib_main_t * vm) /* allocate default region */ error = vlib_physmem_region_alloc (vm, "buffers", vlib_buffer_physmem_sz, 0, - VLIB_PHYSMEM_F_INIT_MHEAP | - VLIB_PHYSMEM_F_HAVE_BUFFERS, - &bm->physmem_region); + VLIB_PHYSMEM_F_INIT_MHEAP, &pri); if (error == 0) - return 0; + goto done; clib_error_free (error); @@ -1098,9 +1110,9 @@ vlib_buffer_main_init (struct vlib_main_t * vm) error = vlib_physmem_region_alloc (vm, "buffers (fake)", vlib_buffer_physmem_sz, 0, VLIB_PHYSMEM_F_FAKE | - VLIB_PHYSMEM_F_INIT_MHEAP | - VLIB_PHYSMEM_F_HAVE_BUFFERS, - &bm->physmem_region); + VLIB_PHYSMEM_F_INIT_MHEAP, &pri); +done: + vlib_buffer_add_physmem_region (vm, pri); return error; } diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index e5c1d21397b..6170323c815 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -123,7 +123,8 @@ typedef struct u8 n_add_refs; /**< Number of additional references to this buffer. */ - u8 dont_waste_me[2]; /**< Available space in the (precious) + u8 buffer_pool_index; /**< index of buffer pool this buffer belongs. */ + u8 dont_waste_me[1]; /**< Available space in the (precious) first 32 octets of buffer metadata Before allocating any of it, discussion required! */ @@ -402,6 +403,14 @@ typedef struct extern vlib_buffer_callbacks_t *vlib_buffer_callbacks; +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + uword start; + uword size; + vlib_physmem_region_index_t physmem_region; +} vlib_buffer_pool_t; + typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); @@ -409,7 +418,7 @@ typedef struct buffer index */ uword buffer_mem_start; uword buffer_mem_size; - vlib_physmem_region_index_t physmem_region; + vlib_buffer_pool_t *buffer_pools; /* Buffer free callback, for subversive activities */ u32 (*buffer_free_callback) (struct vlib_main_t * vm, @@ -442,8 +451,9 @@ typedef struct int callbacks_registered; } vlib_buffer_main_t; -void vlib_buffer_add_mem_range (struct vlib_main_t *vm, uword start, - uword size); +u8 vlib_buffer_add_physmem_region (struct vlib_main_t *vm, + vlib_physmem_region_index_t region); + clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm); typedef struct diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index d51de6bedf3..7224e08013e 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -162,7 +162,10 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) always_inline u64 vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) { - return vlib_physmem_offset_to_physical (vm, vm->buffer_main->physmem_region, + vlib_physmem_region_index_t pri; + vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index); + pri = vm->buffer_main->buffer_pools[b->buffer_pool_index].physmem_region; + return vlib_physmem_offset_to_physical (vm, pri, (((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES) + STRUCT_OFFSET_OF (vlib_buffer_t, diff --git a/src/vlib/linux/physmem.c b/src/vlib/linux/physmem.c index 6d3f7c55d32..dad4ef064b4 100644 --- a/src/vlib/linux/physmem.c +++ b/src/vlib/linux/physmem.c @@ -190,11 +190,6 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, MHEAP_FLAG_THREAD_SAFE); } - if (flags & VLIB_PHYSMEM_F_HAVE_BUFFERS) - { - vlib_buffer_add_mem_range (vm, pointer_to_uword (pr->mem), pr->size); - } - *idx = pr->index; goto done; diff --git a/src/vlib/physmem.h b/src/vlib/physmem.h index a7fed124e27..1e053d65c0f 100644 --- a/src/vlib/physmem.h +++ b/src/vlib/physmem.h @@ -55,7 +55,6 @@ typedef struct void *heap; u32 flags; #define VLIB_PHYSMEM_F_INIT_MHEAP (1<<0) -#define VLIB_PHYSMEM_F_HAVE_BUFFERS (1<<1) #define VLIB_PHYSMEM_F_FAKE (1<<2) u8 numa_node; -- 2.16.6