From 19010202285445372c281faca6a9757fd0ed466c Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 24 Mar 2016 17:17:47 +0100 Subject: [PATCH] Use rte_mempool private data for storing vlib_buffer_t Change-Id: If3fc88a35bc0b736376113a39667caea42802ea1 Signed-off-by: Damjan Marion --- build-data/platforms/virl.mk | 2 -- build-data/platforms/vpp.mk | 2 -- dpdk/Makefile | 2 +- vlib/vlib/buffer.h | 45 ++++++++++++++++++++------------- vlib/vlib/buffer_funcs.h | 37 +++++++++++++++++---------- vlib/vlib/dpdk_buffer.c | 51 +++++++++++++++++++++----------------- vnet/vnet/devices/af_packet/node.c | 8 +++--- vnet/vnet/devices/dpdk/device.c | 20 +++++++-------- vnet/vnet/devices/dpdk/dpdk.h | 1 - vnet/vnet/devices/dpdk/init.c | 4 +-- vnet/vnet/devices/dpdk/node.c | 32 ++++++++++++------------ vnet/vnet/devices/ssvm/ssvm_eth.h | 2 +- vnet/vnet/dpdk_replication.h | 6 ++--- vnet/vnet/ip/ip.h | 2 +- vnet/vnet/ip/ip6_forward.c | 2 +- vnet/vnet/pg/input.c | 6 ++--- vnet/vnet/unix/tapcli.c | 4 +-- vnet/vnet/unix/tuntap.c | 12 +++------ 18 files changed, 126 insertions(+), 112 deletions(-) diff --git a/build-data/platforms/virl.mk b/build-data/platforms/virl.mk index 29b47dad87f..f47f3691ae9 100644 --- a/build-data/platforms/virl.mk +++ b/build-data/platforms/virl.mk @@ -24,8 +24,6 @@ vpp_configure_args_virl = --with-dpdk vnet_configure_args_virl = --with-dpdk --with-virl # Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e. -# dpdk_headroom = uiotarball_headroom = vlib_pre_data + 128 -dpdk_configure_args_virl = --with-headroom=256 vlib_configure_args_virl = --with-pre-data=128 # Override default -march and CONFIG_RTE_MACHINE settings diff --git a/build-data/platforms/vpp.mk b/build-data/platforms/vpp.mk index 6b1ba82f12a..50c85dcc7c1 100644 --- a/build-data/platforms/vpp.mk +++ b/build-data/platforms/vpp.mk @@ -24,8 +24,6 @@ vpp_configure_args_vpp = --with-dpdk vnet_configure_args_vpp = --with-dpdk # Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e. -# dpdk_headroom = uiotarball_headroom = vlib_pre_data + 128 -dpdk_configure_args_vpp = --with-headroom=256 vlib_configure_args_vpp = --with-pre-data=128 diff --git a/dpdk/Makefile b/dpdk/Makefile index a07e86159b2..dabbf6adf48 100644 --- a/dpdk/Makefile +++ b/dpdk/Makefile @@ -16,7 +16,7 @@ SHELL := /bin/bash DPDK_BUILD_DIR ?= $(CURDIR)/_build DPDK_INSTALL_DIR ?= $(CURDIR)/_install -DPDK_PKTMBUF_HEADROOM ?= 256 +DPDK_PKTMBUF_HEADROOM ?= 128 DPDK_DOWNLOAD_DIR ?= $(HOME)/Downloads DPDK_MARCH ?= native DPDK_DEBUG ?= n diff --git a/vlib/vlib/buffer.h b/vlib/vlib/buffer.h index 6322481b696..9c148ef2f21 100644 --- a/vlib/vlib/buffer.h +++ b/vlib/vlib/buffer.h @@ -45,7 +45,16 @@ #include #include #include /* for vlib_error_t */ + +#if DPDK > 0 +#include +#define VLIB_BUFFER_DATA_SIZE (2048) +#define VLIB_BUFFER_PRE_DATA_SIZE RTE_PKTMBUF_HEADROOM +#else #include /* for __PRE_DATA_SIZE */ +#define VLIB_BUFFER_DATA_SIZE (512) +#define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE +#endif #ifdef CLIB_HAVE_VEC128 typedef u8x16 vlib_copy_unit_t; @@ -62,6 +71,7 @@ typedef uword vlib_copy_unit_t; /* VLIB buffer representation. */ typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); /* Offset within data[] that we are currently processing. If negative current header points into predata area. */ i16 current_data; /**< signed offset in data[], pre_data[] @@ -124,23 +134,25 @@ typedef struct { u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes. See .../vnet/vnet/buffer.h */ - /***** end of first cache line */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline1); u32 opaque2[16]; /**< More opaque data, in its own cache line */ /***** end of second cache line */ - u8 pre_data [__PRE_DATA_SIZE]; /**< Space for inserting data - before buffer start. - Packet rewrite string will be - rewritten backwards and may extend - back before buffer->data[0]. - Must come directly before packet data. - */ - -#define VLIB_BUFFER_PRE_DATA_SIZE (ARRAY_LEN (((vlib_buffer_t *)0)->pre_data)) + CLIB_CACHE_LINE_ALIGN_MARK(cacheline2); + u8 pre_data [VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data + before buffer start. + Packet rewrite string will be + rewritten backwards and may extend + back before buffer->data[0]. + Must come directly before packet data. + */ + u8 data[0]; /**< Packet data. Hardware DMA here */ } vlib_buffer_t; /* Must be a multiple of 64B. */ +#define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE) + /** \brief Prefetch buffer metadata. The first 64 bytes of buffer contains most header information @@ -283,13 +295,7 @@ typedef struct { initializing static data for each packet generated. */ vlib_buffer_free_list_t * buffer_free_list_pool; #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0) - -#if DPDK == 1 -/* must be same as dpdk buffer size */ -#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (2048) -#else -#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (512) -#endif +#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE /* Hash table mapping buffer size (rounded to next unit of sizeof (vlib_buffer_t)) to free list index. */ @@ -357,6 +363,11 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m) return sm->tx.n_total_data_bytes + s->current_buffer_index + vec_len (s->overflow_buffer); } +#if DPDK > 0 +#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1) +#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1)) +#endif + /* */ diff --git a/vlib/vlib/buffer_funcs.h b/vlib/vlib/buffer_funcs.h index eea417a94f0..f7bdb12d680 100644 --- a/vlib/vlib/buffer_funcs.h +++ b/vlib/vlib/buffer_funcs.h @@ -254,7 +254,7 @@ u8 * vlib_validate_buffers (vlib_main_t * vm, clib_error_t * vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, - unsigned mbuf_size, unsigned socket_id); + unsigned socket_id); /** \brief Allocate buffers into supplied array @@ -425,11 +425,9 @@ vlib_buffer_chain_init(vlib_buffer_t *first) first->flags &= ~VLIB_BUFFER_NEXT_PRESENT; first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->nb_segs = 1; - (((struct rte_mbuf *) first) - 1)->next = 0; - (((struct rte_mbuf *) first) - 1)->pkt_len = 0; - (((struct rte_mbuf *) first) - 1)->data_len = 0; - (((struct rte_mbuf *) first) - 1)->data_off = RTE_PKTMBUF_HEADROOM + first->current_data; + struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first); + rte_pktmbuf_reset(mb); + mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + first->current_data; #endif } @@ -446,11 +444,17 @@ vlib_buffer_chain_buffer(vlib_main_t *vm, next_buffer->current_length = 0; next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->nb_segs++; - (((struct rte_mbuf *) last) - 1)->next = (((struct rte_mbuf *) next_buffer) - 1); - (((struct rte_mbuf *) next_buffer) - 1)->data_len = 0; - (((struct rte_mbuf *) next_buffer) - 1)->data_off = RTE_PKTMBUF_HEADROOM + next_buffer->current_data; - (((struct rte_mbuf *) next_buffer) - 1)->next = 0; + struct rte_mbuf * mb; + mb = rte_mbuf_from_vlib_buffer(first); + mb->nb_segs++; + + mb = rte_mbuf_from_vlib_buffer(last); + mb->next = rte_mbuf_from_vlib_buffer(next_buffer); + + mb = rte_mbuf_from_vlib_buffer(next_buffer); + mb->data_len = 0; + mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + next_buffer->current_data; + mb->next = 0; #endif return next_buffer; } @@ -468,8 +472,10 @@ vlib_buffer_chain_increase_length(vlib_buffer_t *first, if (first != last) first->total_length_not_including_first_buffer += len; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->pkt_len += len; - (((struct rte_mbuf *) last) - 1)->data_len += len; + struct rte_mbuf * mb_first = rte_mbuf_from_vlib_buffer(first); + struct rte_mbuf * mb_last = rte_mbuf_from_vlib_buffer(last); + mb_first->pkt_len += len; + mb_last->data_len += len; #endif } @@ -589,6 +595,11 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * _dst, vlib_buffer_union_t * dst = (vlib_buffer_union_t *) _dst; vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template; + /* Make sure vlib_buffer_t is cacheline aligned and sized */ + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline0) == 0); + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline1) == CLIB_CACHE_LINE_BYTES); + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline2) == CLIB_CACHE_LINE_BYTES * 2); + /* Make sure buffer template is sane. */ ASSERT (fl->index == fl->buffer_init_template.free_list_index); diff --git a/vlib/vlib/dpdk_buffer.c b/vlib/vlib/dpdk_buffer.c index 04a6447d633..db1fde18c77 100644 --- a/vlib/vlib/dpdk_buffer.c +++ b/vlib/vlib/dpdk_buffer.c @@ -66,12 +66,10 @@ #include -phys_addr_t __attribute__ ((weak)) rte_mem_virt2phy(); -int __attribute__ ((weak)) rte_eal_has_hugepages(); -unsigned __attribute__ ((weak)) rte_socket_id(); -struct rte_mempool * __attribute__ ((weak)) rte_mempool_create(); -void __attribute__ ((weak)) rte_pktmbuf_init(); -void __attribute__ ((weak)) rte_pktmbuf_pool_init(); +#pragma weak rte_mem_virt2phy +#pragma weak rte_eal_has_hugepages +#pragma weak rte_socket_id +#pragma weak rte_pktmbuf_pool_create uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) { @@ -400,13 +398,13 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) for (i = 0; i < vec_len (f->unaligned_buffers); i++) { b = vlib_get_buffer (vm, f->unaligned_buffers[i]); - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } for (i = 0; i < vec_len (f->aligned_buffers); i++) { b = vlib_get_buffer (vm, f->aligned_buffers[i]); - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } @@ -487,7 +485,7 @@ fill_free_list (vlib_main_t * vm, mb->data_off = RTE_PKTMBUF_HEADROOM; mb->nb_segs = 1; - b = (vlib_buffer_t *)(mb+1); + b = vlib_buffer_from_rte_mbuf(mb); bi = vlib_get_buffer_index (vm, b); vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t)); @@ -726,7 +724,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, { if (PREDICT_TRUE (b->clone_count == 0)) { - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } @@ -820,7 +818,7 @@ vlib_packet_template_get_packet (vlib_main_t * vm, /* Fix up mbuf header length fields */ struct rte_mbuf * mb; - mb = ((struct rte_mbuf *)b) - 1; + mb = rte_mbuf_from_vlib_buffer(b); mb->data_len = b->current_length; mb->pkt_len = b->current_length; @@ -916,22 +914,26 @@ vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * b_first) { vlib_buffer_t *b = b_first, *prev = b_first; - struct rte_mbuf *mb_first = ((struct rte_mbuf *) b) - 1; + struct rte_mbuf *mb_prev, *mb, *mb_first; - mb_first->pkt_len = mb_first-> data_len = b_first->current_length; + mb_first = rte_mbuf_from_vlib_buffer(b_first); + + mb_first->pkt_len = mb_first->data_len = b_first->current_length; while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { b = vlib_get_buffer(vm, b->next_buffer); + mb = rte_mbuf_from_vlib_buffer(b); + mb_prev = rte_mbuf_from_vlib_buffer(prev); mb_first->nb_segs++; mb_first->pkt_len += b->current_length; - (((struct rte_mbuf *) prev) - 1)->next = (((struct rte_mbuf *) b) - 1); - (((struct rte_mbuf *) b) - 1)->data_len = b->current_length; + mb_prev->next = mb; + mb->data_len = b->current_length; prev = b; } } clib_error_t * vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, - unsigned mbuf_size, unsigned socket_id) + unsigned socket_id) { vlib_buffer_main_t * bm = vm->buffer_main; vlib_physmem_main_t * vpm = &vm->physmem_main; @@ -939,7 +941,7 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, uword new_start, new_size; int i; - if (!rte_mempool_create) + if (!rte_pktmbuf_pool_create) return clib_error_return (0, "not linked with DPDK"); vec_validate_aligned(bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); @@ -949,12 +951,15 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, return 0; u8 * pool_name = format(0, "mbuf_pool_socket%u%c",socket_id, 0); - rmp = rte_mempool_create((char *) pool_name, - num_mbufs, mbuf_size, 512, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - socket_id, 0); + + rmp = rte_pktmbuf_pool_create((char *) pool_name, /* pool name */ + num_mbufs, /* number of mbufs */ + 512, /* cache size */ + VLIB_BUFFER_HDR_SIZE, /* priv size */ + VLIB_BUFFER_PRE_DATA_SIZE + + VLIB_BUFFER_DATA_SIZE, /* dataroom size */ + socket_id); /* cpu socket */ + vec_free(pool_name); if (rmp) diff --git a/vnet/vnet/devices/af_packet/node.c b/vnet/vnet/devices/af_packet/node.c index 15a96f442f9..efe0e1c2586 100644 --- a/vnet/vnet/devices/af_packet/node.c +++ b/vnet/vnet/devices/af_packet/node.c @@ -94,11 +94,6 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi) vlib_buffer_t * b = vlib_get_buffer (vm, bi); vlib_buffer_t * first_b = vlib_get_buffer (vm, first_bi); vlib_buffer_t * prev_b = vlib_get_buffer (vm, prev_bi); -#if DPDK > 0 - struct rte_mbuf * mbuf = ((struct rte_mbuf *) b) - 1; - struct rte_mbuf * first_mbuf = ((struct rte_mbuf *) first_b) - 1; - struct rte_mbuf * prev_mbuf = ((struct rte_mbuf *) prev_b) - 1; -#endif /* update first buffer */ first_b->total_length_not_including_first_buffer += b->current_length; @@ -111,6 +106,9 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi) b->next_buffer = 0; #if DPDK > 0 + struct rte_mbuf * mbuf = rte_mbuf_from_vlib_buffer(b); + struct rte_mbuf * first_mbuf = rte_mbuf_from_vlib_buffer(first_b); + struct rte_mbuf * prev_mbuf = rte_mbuf_from_vlib_buffer(prev_b); first_mbuf->nb_segs++; prev_mbuf->next = mbuf; mbuf->data_len = b->current_length; diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c index 02703cc59b2..dcf95803c84 100644 --- a/vnet/vnet/devices/dpdk/device.c +++ b/vnet/vnet/devices/dpdk/device.c @@ -87,7 +87,7 @@ static struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b) unsigned socket_id = rte_socket_id(); ASSERT (bm->pktmbuf_pools[socket_id]); - pkt_mb = ((struct rte_mbuf *)b)-1; + pkt_mb = rte_mbuf_from_vlib_buffer(b); nb_segs = pkt_mb->nb_segs; for (nb_segs_left = nb_segs; nb_segs_left; nb_segs_left--) { @@ -159,7 +159,7 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm, dpdk_tx_dma_trace_t * t0; struct rte_mbuf * mb; - mb = ((struct rte_mbuf *)buffer)-1; + mb = rte_mbuf_from_vlib_buffer(buffer); t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0])); t0->queue_index = queue_id; @@ -541,7 +541,7 @@ dpdk_interface_tx (vlib_main_t * vm, { u32 bi0 = from[n_packets]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - struct rte_mbuf *mb0 = ((struct rte_mbuf *)b0) - 1; + struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer(b0); rte_pktmbuf_free (mb0); } return n_on_ring; @@ -584,9 +584,9 @@ dpdk_interface_tx (vlib_main_t * vm, pref0 = vlib_get_buffer (vm, pi0); pref1 = vlib_get_buffer (vm, pi1); - prefmb0 = ((struct rte_mbuf *)pref0) - 1; - prefmb1 = ((struct rte_mbuf *)pref1) - 1; - + prefmb0 = rte_mbuf_from_vlib_buffer(pref0); + prefmb1 = rte_mbuf_from_vlib_buffer(pref1); + CLIB_PREFETCH(prefmb0, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH(pref0, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH(prefmb1, CLIB_CACHE_LINE_BYTES, LOAD); @@ -599,8 +599,8 @@ dpdk_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); - mb0 = ((struct rte_mbuf *)b0) - 1; - mb1 = ((struct rte_mbuf *)b1) - 1; + mb0 = rte_mbuf_from_vlib_buffer(b0); + mb1 = rte_mbuf_from_vlib_buffer(b1); any_clone = b0->clone_count | b1->clone_count; if (PREDICT_FALSE(any_clone != 0)) @@ -701,7 +701,7 @@ dpdk_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); - mb0 = ((struct rte_mbuf *)b0) - 1; + mb0 = rte_mbuf_from_vlib_buffer(b0); if (PREDICT_FALSE(b0->clone_count != 0)) { struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0); @@ -921,7 +921,7 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) vlib_buffer_main_t * bm = vm->buffer_main; memset(&conf, 0, sizeof(conf)); snprintf(conf.name, RTE_KNI_NAMESIZE, "vpp%u", xd->kni_port_id); - conf.mbuf_size = MBUF_SIZE; + conf.mbuf_size = VLIB_BUFFER_DATA_SIZE; memset(&ops, 0, sizeof(ops)); ops.port_id = xd->kni_port_id; ops.change_mtu = kni_change_mtu; diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h index 656f39ea681..14f7b3e8518 100644 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ b/vnet/vnet/devices/dpdk/dpdk.h @@ -62,7 +62,6 @@ #define always_inline static inline __attribute__ ((__always_inline__)) #endif -#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) #define NB_MBUF (32<<10) extern vnet_device_class_t dpdk_device_class; diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c index 8fe95ae5f09..8bb253a3a66 100644 --- a/vnet/vnet/devices/dpdk/init.c +++ b/vnet/vnet/devices/dpdk/init.c @@ -1405,13 +1405,13 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) rte_dump_physmem_layout(stdout); /* main thread 1st */ - error = vlib_buffer_pool_create(vm, dm->num_mbufs, MBUF_SIZE, rte_socket_id()); + error = vlib_buffer_pool_create(vm, dm->num_mbufs, rte_socket_id()); if (error) return error; for (i = 0; i < RTE_MAX_LCORE; i++) { - error = vlib_buffer_pool_create(vm, dm->num_mbufs, MBUF_SIZE, + error = vlib_buffer_pool_create(vm, dm->num_mbufs, rte_lcore_to_socket_id(i)); if (error) return error; diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c index 72d564ba8b2..4f5a84ae29a 100644 --- a/vnet/vnet/devices/dpdk/node.c +++ b/vnet/vnet/devices/dpdk/node.c @@ -366,7 +366,7 @@ void dpdk_rx_trace (dpdk_main_t * dm, n_left -= 1; b0 = vlib_get_buffer (vm, bi0); - mb = ((struct rte_mbuf *)b0) - 1; + mb = rte_mbuf_from_vlib_buffer(b0); dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0, &next0, &error0); vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0); @@ -602,20 +602,20 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, if (PREDICT_TRUE(n_buffers > 2)) { struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2]; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, STORE); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); } ASSERT(mb); - b0 = (vlib_buffer_t *)(mb+1); + b0 = vlib_buffer_from_rte_mbuf(mb); /* check whether EFD is looking for packets to discard */ if (PREDICT_FALSE(efd_discard_burst)) { vlib_thread_main_t * tm = vlib_get_thread_main(); - + if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb))) { rte_pktmbuf_free(mb); @@ -633,7 +633,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, if (PREDICT_FALSE(mb->nb_segs > 1)) { struct rte_mbuf *pfmb = mb->next; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); b_chain = b0; @@ -693,7 +693,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, { ASSERT(mb_seg != 0); - b_seg = (vlib_buffer_t *)(mb_seg+1); + b_seg = vlib_buffer_from_rte_mbuf(mb_seg); vlib_buffer_init_for_free_list (b_seg, fl); b_seg->clone_count = 0; @@ -1280,13 +1280,13 @@ void dpdk_io_thread (vlib_worker_thread_t * w, if (PREDICT_TRUE(n_buffers > 1)) { struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2]; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD); } - b0 = (vlib_buffer_t *)(mb+1); + b0 = vlib_buffer_from_rte_mbuf(mb); /* check whether EFD is looking for packets to discard */ if (PREDICT_FALSE(efd_discard_burst)) @@ -1310,7 +1310,7 @@ void dpdk_io_thread (vlib_worker_thread_t * w, if (PREDICT_FALSE(mb->nb_segs > 1)) { struct rte_mbuf *pfmb = mb->next; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); b_chain = b0; @@ -1361,7 +1361,7 @@ void dpdk_io_thread (vlib_worker_thread_t * w, { ASSERT(mb_seg != 0); - b_seg = (vlib_buffer_t *)(mb_seg+1); + b_seg = vlib_buffer_from_rte_mbuf(mb_seg); vlib_buffer_init_for_free_list (b_seg, fl); b_seg->clone_count = 0; @@ -1681,14 +1681,14 @@ dpdk_io_input (vlib_main_t * vm, if (PREDICT_TRUE(n_buffers > 1)) { struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2]; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD); } - - b0 = (vlib_buffer_t *)(mb+1); - + + b0 = vlib_buffer_from_rte_mbuf(mb); + /* check whether EFD is looking for packets to discard */ if (PREDICT_FALSE(efd_discard_burst)) { @@ -1711,7 +1711,7 @@ dpdk_io_input (vlib_main_t * vm, if (PREDICT_FALSE(mb->nb_segs > 1)) { struct rte_mbuf *pfmb = mb->next; - vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1); + vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb); CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE); b_chain = b0; @@ -1762,7 +1762,7 @@ dpdk_io_input (vlib_main_t * vm, { ASSERT(mb_seg != 0); - b_seg = (vlib_buffer_t *)(mb_seg+1); + b_seg = vlib_buffer_from_rte_mbuf(mb_seg); vlib_buffer_init_for_free_list (b_seg, fl); b_seg->clone_count = 0; diff --git a/vnet/vnet/devices/ssvm/ssvm_eth.h b/vnet/vnet/devices/ssvm/ssvm_eth.h index 70d895b9ad4..8f1f88964c2 100644 --- a/vnet/vnet/devices/ssvm/ssvm_eth.h +++ b/vnet/vnet/devices/ssvm/ssvm_eth.h @@ -35,7 +35,7 @@ extern vnet_device_class_t ssvm_eth_device_class; extern vlib_node_registration_t ssvm_eth_input_node; #define SSVM_BUFFER_SIZE \ - (VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES + VLIB_BUFFER_PRE_DATA_SIZE) + (VLIB_BUFFER_DATA_SIZE + VLIB_BUFFER_PRE_DATA_SIZE) #define SSVM_PACKET_TYPE 1 typedef struct { diff --git a/vnet/vnet/dpdk_replication.h b/vnet/vnet/dpdk_replication.h index b25558f9b89..6259c449af0 100644 --- a/vnet/vnet/dpdk_replication.h +++ b/vnet/vnet/dpdk_replication.h @@ -49,7 +49,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b) return 0; src_buf = b; - rv = dst_buf = (vlib_buffer_t *)(rte_mbufs[0] + 1); + rv = dst_buf = vlib_buffer_from_rte_mbuf(rte_mbufs[0]); vlib_buffer_init_for_free_list (dst_buf, fl); copy_src = b->data + src_buf->current_data; copy_dst = dst_buf->data + src_buf->current_data; @@ -75,7 +75,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b) if (i < new_buffers_needed - 1) { src_buf = vlib_get_buffer (vm, src_buf->next_buffer); - dst_buf = (vlib_buffer_t *)(rte_mbufs[i+1] + 1); + dst_buf = vlib_buffer_from_rte_mbuf(rte_mbufs[i+1]); vlib_buffer_init_for_free_list (dst_buf, fl); copy_src = src_buf->data; copy_dst = dst_buf->data; @@ -87,7 +87,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b) if (rte_mempool_get_bulk (rmp, (void **)rte_mbufs, 1) < 0) return 0; - rv = (vlib_buffer_t *)(rte_mbufs[0] + 1); + rv = vlib_buffer_from_rte_mbuf(rte_mbufs[0]); vlib_buffer_init_for_free_list (rv, fl); memcpy(rv->data + b->current_data, b->data + b->current_data, diff --git a/vnet/vnet/ip/ip.h b/vnet/vnet/ip/ip.h index 76a2552ffa4..de46ad38c5f 100644 --- a/vnet/vnet/ip/ip.h +++ b/vnet/vnet/ip/ip.h @@ -162,7 +162,7 @@ ip_incremental_checksum_buffer (vlib_main_t * vm, vlib_buffer_t * first_buffer, #if DPDK > 0 { u32 n_bytes_left = n_bytes_to_checksum; - struct rte_mbuf * mb = ((struct rte_mbuf *)first_buffer)-1; + struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first_buffer); u8 nb_segs = mb->nb_segs; ASSERT(mb->data_len >= first_buffer_offset); void * h; diff --git a/vnet/vnet/ip/ip6_forward.c b/vnet/vnet/ip/ip6_forward.c index 1d0e21e528c..a478bab5fc4 100644 --- a/vnet/vnet/ip/ip6_forward.c +++ b/vnet/vnet/ip/ip6_forward.c @@ -1445,7 +1445,7 @@ u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6 #if DPDK > 0 if (p0) { - struct rte_mbuf *mb = ((struct rte_mbuf *)p0)-1; + struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0); u8 nb_segs = mb->nb_segs; n_this_buffer = (p0->current_length > headers_size ? diff --git a/vnet/vnet/pg/input.c b/vnet/vnet/pg/input.c index 4ec61ca7b96..38402c2e989 100644 --- a/vnet/vnet/pg/input.c +++ b/vnet/vnet/pg/input.c @@ -61,7 +61,7 @@ pg_set_mbuf_metadata (pg_main_t * pg, u32 * buffers, u32 n_alloc) for (i = 0; i < n_alloc; i++) { b = vlib_get_buffer (vm, buffers[i]); - mb = ((struct rte_mbuf *)b) - 1; + mb = rte_mbuf_from_vlib_buffer(b); delta = vlib_buffer_length_in_chain (vm, b) - (i16) mb->pkt_len; new_data_len = (u16)((i16) mb->data_len + delta); @@ -1473,9 +1473,9 @@ pg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers) ({ vlib_buffer_t * b; struct rte_mbuf *mb; - + b = vlib_get_buffer(vm, bi0[0]); - mb = (struct rte_mbuf *)b - 1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); })); } diff --git a/vnet/vnet/unix/tapcli.c b/vnet/vnet/unix/tapcli.c index 5b0ac9378ee..7776b9ee837 100644 --- a/vnet/vnet/unix/tapcli.c +++ b/vnet/vnet/unix/tapcli.c @@ -220,7 +220,7 @@ static uword tapcli_rx_iface(vlib_main_t * vm, tapcli_interface_t * ti) { tapcli_main_t * tm = &tapcli_main; - const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; + const uword buffer_size = VLIB_BUFFER_DATA_SIZE; u32 n_trace = vlib_get_trace_count (vm, node); u8 set_trace = 0; @@ -434,7 +434,7 @@ static clib_error_t * tapcli_config (vlib_main_t * vm, unformat_input_t * input) { tapcli_main_t *tm = &tapcli_main; - const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; + const uword buffer_size = VLIB_BUFFER_DATA_SIZE; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { diff --git a/vnet/vnet/unix/tuntap.c b/vnet/vnet/unix/tuntap.c index 77c60fd6ee2..7ea0b70348d 100644 --- a/vnet/vnet/unix/tuntap.c +++ b/vnet/vnet/unix/tuntap.c @@ -200,12 +200,11 @@ tuntap_rx (vlib_main_t * vm, tuntap_main_t * tm = &tuntap_main; vlib_buffer_t * b; u32 bi; + const uword buffer_size = VLIB_BUFFER_DATA_SIZE; #if DPDK == 0 - const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; u32 free_list_index = VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX; #else dpdk_main_t * dm = &dpdk_main; - const uword buffer_size = MBUF_SIZE; u32 free_list_index = dm->vlib_buffer_free_list_index; #endif @@ -262,7 +261,7 @@ tuntap_rx (vlib_main_t * vm, #endif b = vlib_get_buffer (vm, tm->rx_buffers[i_rx]); #if DPDK == 1 - mb = (((struct rte_mbuf *)b)-1); + mb = rte_mbuf_from_vlib_buffer(b); #endif b->flags = 0; b->current_data = 0; @@ -445,12 +444,7 @@ tuntap_config (vlib_main_t * vm, unformat_input_t * input) u8 * name; int flags = IFF_TUN | IFF_NO_PI; int is_enabled = 0, is_ether = 0, have_normal_interface = 0; -#if DPDK == 0 - const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; -#else - const uword buffer_size = MBUF_SIZE; -#endif - + const uword buffer_size = VLIB_BUFFER_DATA_SIZE; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { -- 2.16.6