From a7e830ed468f35590871988e68153222325870b8 Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Fri, 23 Apr 2021 15:16:50 +0200 Subject: [PATCH] vlib: move offload flags to 1st cacheline in vlib_buffer_t Type: improvement Some tests i.e. ipsec see performance regression when offload flags are moved to 2nd cacheline. This patch moves them back to 1st cacheline. Change-Id: I6ead45ff6d2c467b0d248f409e27c2ba31758741 Signed-off-by: Mohsin Kazmi --- src/plugins/avf/output.c | 2 +- src/plugins/dpdk/device/device.c | 2 +- src/plugins/gtpu/gtpu_decap.c | 4 ++-- src/vnet/buffer.c | 2 +- src/vnet/buffer.h | 14 ++++++-------- src/vnet/devices/virtio/device.c | 4 ++-- src/vnet/devices/virtio/vhost_user_output.c | 2 +- src/vnet/interface_output.h | 3 ++- src/vnet/ip/ip4_forward.c | 4 ++-- src/vnet/ip/ip6_forward.c | 6 +++--- src/vnet/ipsec/ipsec_output.c | 2 +- 11 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index e16d79dfc6d..f35f7bb962b 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -60,7 +60,7 @@ avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso) if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD)) return 0; - u32 oflags = vnet_buffer2 (b)->oflags; + u32 oflags = vnet_buffer (b)->oflags; u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 94b3f6b289a..7e0f2d43994 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -228,7 +228,7 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0)) return; - oflags = vnet_buffer2 (b)->oflags; + oflags = vnet_buffer (b)->oflags; ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM; tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c index 5657c08031b..40243dbcc53 100644 --- a/src/plugins/gtpu/gtpu_decap.c +++ b/src/plugins/gtpu/gtpu_decap.c @@ -1256,12 +1256,12 @@ static char *gtpu_flow_error_strings[] = { #define gtpu_local_need_csum_check(_b) \ (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \ (_b->flags & VNET_BUFFER_F_OFFLOAD && \ - vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) + vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) #define gtpu_local_csum_is_valid(_b) \ ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \ (_b->flags & VNET_BUFFER_F_OFFLOAD && \ - vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0) + vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0) static_always_inline u8 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b) diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c index b41036d8124..545018a79e9 100644 --- a/src/vnet/buffer.c +++ b/src/vnet/buffer.c @@ -22,7 +22,7 @@ format_vnet_buffer_offload (u8 *s, va_list *args) vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); #define _(bit, name, ss, v) \ - if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \ + if (v && (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \ s = format (s, "%s ", ss); foreach_vnet_buffer_offload_flag #undef _ diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index 19585e22506..0816232aed7 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -138,7 +138,8 @@ typedef struct i16 l3_hdr_offset; i16 l4_hdr_offset; u8 feature_arc_index; - u8 dont_waste_me; + /* offload flags */ + u8 oflags; union { @@ -472,9 +473,6 @@ typedef struct u16 gso_size; /* size of L4 prototol header */ u16 gso_l4_hdr_sz; - - /* offload flags */ - u32 oflags; }; struct @@ -526,12 +524,12 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags) if (b->flags & VNET_BUFFER_F_OFFLOAD) { /* add a flag to existing offload */ - vnet_buffer2 (b)->oflags |= oflags; + vnet_buffer (b)->oflags |= oflags; } else { /* no offload yet: reset offload flags to new value */ - vnet_buffer2 (b)->oflags = oflags; + vnet_buffer (b)->oflags = oflags; b->flags |= VNET_BUFFER_F_OFFLOAD; } } @@ -539,8 +537,8 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags) static_always_inline void vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags) { - vnet_buffer2 (b)->oflags &= ~oflags; - if (0 == vnet_buffer2 (b)->oflags) + vnet_buffer (b)->oflags &= ~oflags; + if (0 == vnet_buffer (b)->oflags) b->flags &= ~VNET_BUFFER_F_OFFLOAD; } diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 237e3d91f1a..20264787292 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -285,7 +285,7 @@ static void set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2) { - u32 oflags = vnet_buffer2 (b)->oflags; + u32 oflags = vnet_buffer (b)->oflags; if (b->flags & VNET_BUFFER_F_IS_IP4) { @@ -334,7 +334,7 @@ set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, static void set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2) { - u32 oflags = vnet_buffer2 (b)->oflags; + u32 oflags = vnet_buffer (b)->oflags; if (b->flags & VNET_BUFFER_F_IS_IP4) { diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index 8dbddea46f5..6406fe3f587 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -225,7 +225,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, generic_header_offset_t gho = { 0 }; int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6; - u32 oflags = vnet_buffer2 (b)->oflags; + u32 oflags = vnet_buffer (b)->oflags; ASSERT (!(is_ip4 && is_ip6)); vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6); diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h index bfd2066c2a8..bb27cdb616e 100644 --- a/src/vnet/interface_output.h +++ b/src/vnet/interface_output.h @@ -85,7 +85,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b, ip6_header_t *ip6; tcp_header_t *th; udp_header_t *uh; - u32 oflags = vnet_buffer2 (b)->oflags; + u32 oflags; if (!(b->flags & VNET_BUFFER_F_OFFLOAD)) return; @@ -96,6 +96,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b, ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset); th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset); uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset); + oflags = vnet_buffer (b)->oflags; if (is_ip4) { diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index ddf97d48a15..da8522de775 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1419,7 +1419,7 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p, #define ip4_local_csum_is_offloaded(_b) \ ((_b->flags & VNET_BUFFER_F_OFFLOAD) && \ - (vnet_buffer2 (_b)->oflags & \ + (vnet_buffer (_b)->oflags & \ (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) #define ip4_local_need_csum_check(is_tcp_udp, _b) \ @@ -2067,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next, /* Verify checksum. */ ASSERT (ip4_header_checksum_is_valid (ip) || - (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)); + (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)); } always_inline uword diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index 0c1bc3b33ec..4f1b148c365 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -1334,8 +1334,8 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, flags[1] = b[1]->flags; u32 oflags[2]; - oflags[0] = vnet_buffer2 (b[0])->oflags; - oflags[1] = vnet_buffer2 (b[1])->oflags; + oflags[0] = vnet_buffer (b[0])->oflags; + oflags[1] = vnet_buffer (b[1])->oflags; u32 l4_offload[2]; l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) && @@ -1524,7 +1524,7 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 flags = b[0]->flags; - u32 oflags = vnet_buffer2 (b[0])->oflags; + u32 oflags = vnet_buffer (b[0])->oflags; u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) && (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c index 527aea5d11d..598195f2737 100644 --- a/src/vnet/ipsec/ipsec_output.c +++ b/src/vnet/ipsec/ipsec_output.c @@ -309,7 +309,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD)) { - u32 oflags = vnet_buffer2 (b0)->oflags; + u32 oflags = vnet_buffer (b0)->oflags; /* * Clearing offload flags before checksum is computed -- 2.16.6