interface loop0
tx-interface loop1
node loop1-output
- buffer-flags ip4 offload-ip-cksum offload-udp-cksum
+ buffer-flags ip4 offload
+ buffer-offload-flags offload-ip-cksum offload-udp-cksum
data { IP4: 1.2.3 -> 4.5.6
UDP: 11.22.33.44 -> 11.22.34.44
ttl 2 checksum 13
avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
{
u64 flags = 0;
- if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
return 0;
- u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+
+ u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+ u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
ASSERT (!is_tcp || !is_udp);
{
u16 next = txq->next;
u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
- const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_GSO;
+ const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO;
u64 one_by_one_offload_flags = 0;
int is_tso;
u16 n_desc = 0;
dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
struct rte_mbuf *mb)
{
- u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
- u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len;
+ u32 oflags, ip_cksum, tcp_cksum, udp_cksum;
u64 ol_flags;
/* Is there any work for us? */
- if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
+ if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
return;
+ oflags = vnet_buffer2 (b)->oflags;
+ ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
+ tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+ udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
+
mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
vnet_buffer (b)->l3_hdr_offset;
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
- (or_flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ (or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
- (or_flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ (or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);
};
-#define gtpu_local_need_csum_check(_b) \
- (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
-
-#define gtpu_local_csum_is_valid(_b) \
- ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0)
+#define gtpu_local_need_csum_check(_b) \
+ (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \
+ (_b->flags & VNET_BUFFER_F_OFFLOAD && \
+ vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+
+#define gtpu_local_csum_is_valid(_b) \
+ ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \
+ (_b->flags & VNET_BUFFER_F_OFFLOAD && \
+ vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
static_always_inline u8
gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
if (csum_offload)
{
ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1);
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data;
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
uh->checksum = 0;
u16 gso_size)
{
u8 l4_hdr_sz = 0;
+ u32 oflags = 0;
if (rx_comp->flags & VMXNET3_RXCF_IP4)
{
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
ip4_header_bytes (ip4);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
{
if (!(rx_comp->flags & VMXNET3_RXCF_IPC))
{
- hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
ip4->checksum = 0;
}
if (!(rx_comp->flags & VMXNET3_RXCF_TUC))
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
sizeof (ip6_header_t);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
hb->flags |= VNET_BUFFER_F_GSO;
}
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (hb, oflags);
}
static_always_inline uword
clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len);
b[0]->current_length = decr_len;
- b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_clear (b[0],
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
wg_timers_any_authenticated_packet_received (peer);
wg_timers_any_authenticated_packet_traversal (peer);
#include <vlib/vlib.h>
#include <vnet/buffer.h>
+u8 *
+format_vnet_buffer_offload (u8 *s, va_list *args)
+{
+ vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+
+#define _(bit, name, ss, v) \
+ if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
+ s = format (s, "%s ", ss);
+ foreach_vnet_buffer_offload_flag
+#undef _
+ return s;
+}
u8 *
format_vnet_buffer (u8 * s, va_list * args)
a = format (a, "%s ", ss);
foreach_vnet_buffer_flag
#undef _
- if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
+ if (b->flags & VNET_BUFFER_F_OFFLOAD) a =
+ format (a, "%U ", format_vnet_buffer_offload, b);
+
+ if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
a = format (a, "l2-hdr-offset %d ", vnet_buffer (b)->l2_hdr_offset);
if (b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
if (b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID)
a = format (a, "l4-hdr-offset %d ", vnet_buffer (b)->l4_hdr_offset);
+ if (b->flags & VNET_BUFFER_F_GSO)
+ a = format (a, "gso gso-size %d", vnet_buffer2 (b)->gso_size);
+
if (b->flags & VNET_BUFFER_F_QOS_DATA_VALID)
a = format (a, "qos %d.%d ",
vnet_buffer2 (b)->qos.bits, vnet_buffer2 (b)->qos.source);
* Flags that are set in the high order bits of ((vlib_buffer*)b)->flags
*
*/
-#define foreach_vnet_buffer_flag \
- _( 1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
- _( 2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
- _( 3, VLAN_2_DEEP, "vlan-2-deep", 1) \
- _( 4, VLAN_1_DEEP, "vlan-1-deep", 1) \
- _( 5, SPAN_CLONE, "span-clone", 1) \
- _( 6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
- _( 7, LOCALLY_ORIGINATED, "local", 1) \
- _( 8, IS_IP4, "ip4", 1) \
- _( 9, IS_IP6, "ip6", 1) \
- _(10, OFFLOAD_IP_CKSUM, "offload-ip-cksum", 1) \
- _(11, OFFLOAD_TCP_CKSUM, "offload-tcp-cksum", 1) \
- _(12, OFFLOAD_UDP_CKSUM, "offload-udp-cksum", 1) \
- _(13, IS_NATED, "natted", 1) \
- _(14, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
- _(15, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
- _(16, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
- _(17, FLOW_REPORT, "flow-report", 1) \
- _(18, IS_DVR, "dvr", 1) \
- _(19, QOS_DATA_VALID, "qos-data-valid", 0) \
- _(20, GSO, "gso", 0) \
- _(21, AVAIL1, "avail1", 1) \
- _(22, AVAIL2, "avail2", 1) \
- _(23, AVAIL3, "avail3", 1) \
- _(24, AVAIL4, "avail4", 1) \
- _(25, AVAIL5, "avail5", 1) \
- _(26, AVAIL6, "avail6", 1) \
- _(27, AVAIL7, "avail7", 1)
+#define foreach_vnet_buffer_flag \
+ _ (1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
+ _ (2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
+ _ (3, VLAN_2_DEEP, "vlan-2-deep", 1) \
+ _ (4, VLAN_1_DEEP, "vlan-1-deep", 1) \
+ _ (5, SPAN_CLONE, "span-clone", 1) \
+ _ (6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
+ _ (7, LOCALLY_ORIGINATED, "local", 1) \
+ _ (8, IS_IP4, "ip4", 1) \
+ _ (9, IS_IP6, "ip6", 1) \
+ _ (10, OFFLOAD, "offload", 0) \
+ _ (11, IS_NATED, "natted", 1) \
+ _ (12, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
+ _ (13, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
+ _ (14, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
+ _ (15, FLOW_REPORT, "flow-report", 1) \
+ _ (16, IS_DVR, "dvr", 1) \
+ _ (17, QOS_DATA_VALID, "qos-data-valid", 0) \
+ _ (18, GSO, "gso", 0) \
+ _ (19, AVAIL1, "avail1", 1) \
+ _ (20, AVAIL2, "avail2", 1) \
+ _ (21, AVAIL3, "avail3", 1) \
+ _ (22, AVAIL4, "avail4", 1) \
+ _ (23, AVAIL5, "avail5", 1) \
+ _ (24, AVAIL6, "avail6", 1) \
+ _ (25, AVAIL7, "avail7", 1) \
+ _ (26, AVAIL8, "avail8", 1) \
+ _ (27, AVAIL9, "avail9", 1)
/*
* Please allocate the FIRST available bit, redefine
* VNET_BUFFER_FLAGS_ALL_AVAIL definition.
*/
-#define VNET_BUFFER_FLAGS_ALL_AVAIL \
- (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
- VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
- VNET_BUFFER_F_AVAIL7)
+#define VNET_BUFFER_FLAGS_ALL_AVAIL \
+ (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
+ VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
+ VNET_BUFFER_F_AVAIL7 | VNET_BUFFER_F_AVAIL8 | VNET_BUFFER_F_AVAIL9)
#define VNET_BUFFER_FLAGS_VLAN_BITS \
(VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
#define vnet_buffer(b) ((vnet_buffer_opaque_t *) (b)->opaque)
+#define foreach_vnet_buffer_offload_flag \
+ _ (0, IP_CKSUM, "offload-ip-cksum", 1) \
+ _ (1, TCP_CKSUM, "offload-tcp-cksum", 1) \
+ _ (2, UDP_CKSUM, "offload-udp-cksum", 1) \
+ _ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1) \
+ _ (4, OUTER_TCP_CKSUM, "offload-outer-tcp-cksum", 1) \
+ _ (5, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1)
+
+enum
+{
+#define _(bit, name, s, v) VNET_BUFFER_OFFLOAD_F_##name = (1 << bit),
+ foreach_vnet_buffer_offload_flag
+#undef _
+};
+
/* Full cache line (64 bytes) of additional space */
typedef struct
{
* in case the egress interface is not GSO-enabled - then we need to perform
* the segmentation, and use this value to cut the payload appropriately.
*/
- u16 gso_size;
- /* size of L4 prototol header */
- u16 gso_l4_hdr_sz;
+ struct
+ {
+ u16 gso_size;
+ /* size of L4 prototol header */
+ u16 gso_l4_hdr_sz;
- /* The union below has a u64 alignment, so this space is unused */
- u32 __unused2[1];
+ /* offload flags */
+ u32 oflags;
+ };
struct
{
format_function_t format_vnet_buffer;
+format_function_t format_vnet_buffer_offload;
+
+static_always_inline void
+vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
+{
+ vnet_buffer2 (b)->oflags |= oflags;
+ b->flags |= VNET_BUFFER_F_OFFLOAD;
+}
+
+static_always_inline void
+vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
+{
+ vnet_buffer2 (b)->oflags &= ~oflags;
+ if (0 == vnet_buffer2 (b)->oflags)
+ b->flags &= ~VNET_BUFFER_F_OFFLOAD;
+}
#endif /* included_vnet_buffer_h */
mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
{
ethernet_header_t *eth = vlib_buffer_get_current (b);
+ u32 oflags = 0;
if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
{
ip4_header_t *ip4 =
b->flags |= VNET_BUFFER_F_IS_IP4;
if (ip4->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
}
else if (ip4->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
{
}
if (ip6->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp =
(tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
}
else if (ip6->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp =
(udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip6_hdr_len;
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
}
set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
0 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
*/
ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
1 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
* virtio devices do not support IP4 checksum offload. So driver takes care
* of it while doing tx.
*/
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
u16 ethertype = 0, l2hdr_sz = 0;
+ u32 oflags = 0;
if (type == VIRTIO_IF_TYPE_TUN)
{
(ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
*l4_proto = ip4->protocol;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |=
- (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
}
if (*l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
}
else if (*l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
*l4_hdr_sz = sizeof (*udp);
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
+ u32 oflags = 0;
if (ethernet_frame_is_tagged (ethertype))
{
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
- b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ b0->flags |= VNET_BUFFER_F_IS_IP4;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
l4_hdr_sz = sizeof (udp_header_t);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
static_always_inline void
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
}
/* checksum offload */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
- (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags
{
u32 flags = 0;
- if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (b0->flags & VNET_BUFFER_F_OFFLOAD)
return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
vlib_buffer_advance (b0, gho0->l3_hdr_offset);
if (is_ip4)
ip4->length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l3_hdr_offset);
- b0->flags |=
- (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+ b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
+ vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
else if (gho0.gho_flags & GHO_F_IP6)
{
ip6->payload_length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l4_hdr_offset);
- b0->flags |=
- (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
+ vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
tcp_header_t *tcp0 =
{
udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ /* FIXME: it should be OUTER_UDP_CKSUM */
+ vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
}
}
tcp->checksum = 0;
tcp->checksum =
ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer_offload_flags_clear (b0,
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
}
else
tcp->checksum = 0;
tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4);
}
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
/* Red herring, there is no such buffer flag */
- if (unformat (input, "avail8"))
+ if (unformat (input, "avail10"))
return 0;
#define _(bit,enum,str,verbose) \
else if (unformat (input, str)) \
return rv;
}
+uword
+unformat_vnet_buffer_offload_flags (unformat_input_t *input, va_list *args)
+{
+ u32 *flagp = va_arg (*args, u32 *);
+ int rv = 0;
+ u32 oflags = 0;
+
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (0)
+ ;
+#define _(bit, enum, str, verbose) \
+ else if (unformat (input, str)) \
+ { \
+ oflags |= (1 << bit); \
+ rv = 1; \
+ }
+ foreach_vnet_buffer_offload_flag
+#undef _
+ else break;
+ }
+ if (rv)
+ *flagp = oflags;
+ return rv;
+}
+
uword
unformat_vnet_hw_interface (unformat_input_t * input, va_list * args)
{
unformat_function_t unformat_vnet_sw_interface;
unformat_function_t unformat_vnet_hw_interface;
unformat_function_t unformat_vnet_buffer_flags;
+unformat_function_t unformat_vnet_buffer_offload_flags;
/* Parses interface flags (up, down, enable, disable, etc.) */
unformat_function_t unformat_vnet_hw_interface_flags;
if (do_tx_offloads)
{
- u32 vnet_buffer_offload_flags =
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- if (or_flags & vnet_buffer_offload_flags)
+ if (or_flags & VNET_BUFFER_F_OFFLOAD)
{
- if (b[0]->flags & vnet_buffer_offload_flags)
+ if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,
b[0]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[1]->flags & vnet_buffer_offload_flags)
+ if (b[1]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[1],
b[1]->flags & VNET_BUFFER_F_IS_IP4,
b[1]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[2]->flags & vnet_buffer_offload_flags)
+ if (b[2]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[2],
b[2]->flags & VNET_BUFFER_F_IS_IP4,
b[2]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[3]->flags & vnet_buffer_offload_flags)
+ if (b[3]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[3],
b[3]->flags & VNET_BUFFER_F_IS_IP4,
if (do_tx_offloads)
{
- if (b[0]->flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+ if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,
#include <vnet/gso/hdr_offset_parser.h>
static_always_inline void
-vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
- ip4_header_t * ip4, tcp_header_t * th,
- udp_header_t * uh)
+vnet_calc_ip4_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip4_header_t *ip4,
+ tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
static_always_inline void
-vnet_calc_ip6_checksums (vlib_main_t * vm, vlib_buffer_t * b,
- ip6_header_t * ip6, tcp_header_t * th,
- udp_header_t * uh)
+vnet_calc_ip6_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip6_header_t *ip6,
+ tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
int bogus;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
}
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
ip6_header_t *ip6;
tcp_header_t *th;
udp_header_t *uh;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
if (is_ip4)
{
- vnet_calc_ip4_checksums (vm, b, ip4, th, uh);
+ vnet_calc_ip4_checksums (vm, b, ip4, th, uh, oflags);
}
else if (is_ip6)
{
- vnet_calc_ip6_checksums (vm, b, ip6, th, uh);
+ vnet_calc_ip6_checksums (vm, b, ip6, th, uh, oflags);
}
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (b, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
#endif
}
}
-#define ip4_local_csum_is_offloaded(_b) \
- _b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM
+#define ip4_local_csum_is_offloaded(_b) \
+ ((_b->flags & VNET_BUFFER_F_OFFLOAD) && \
+ (vnet_buffer2 (_b)->oflags & \
+ (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define ip4_local_need_csum_check(is_tcp_udp, _b) \
(is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
/* Verify checksum. */
ASSERT (ip4_header_checksum_is_valid (ip) ||
- (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
+ (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
if (csum_offload)
{
ih->checksum = 0;
- b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
}
else
ih->checksum = ip4_header_checksum (ih);
flags[0] = b[0]->flags;
flags[1] = b[1]->flags;
+ u32 oflags[2];
+ oflags[0] = vnet_buffer2 (b[0])->oflags;
+ oflags[1] = vnet_buffer2 (b[1])->oflags;
+
+ u32 l4_offload[2];
+ l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+ l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
u32 good_l4_csum[2];
good_l4_csum[0] =
- flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
good_l4_csum[1] =
- flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
u32 udp_offset[2] = { };
u8 is_tcp_udp[2];
u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
u32 flags = b[0]->flags;
- u32 good_l4_csum =
- flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ u32 oflags = vnet_buffer2 (b[0])->oflags;
+
+ u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
+ u32 good_l4_csum =
+ (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
u32 udp_offset;
i16 len_diff = 0;
u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
- u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
- && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
+ u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
+ !good_l4_csum &&
+ !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
if (PREDICT_FALSE (need_csum))
{
flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
-
STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
IP6_ERROR_UDP_CHECKSUM,
"Wrong IP6 errors constants");
if (p1->flags & VNET_BUFFER_F_GSO)
ip1_len = gso_mtu_sz (p1);
-
-
ip6_mtu_check (p0, ip0_len,
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
adj1[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated1, &next1, is_midchain,
&error1);
-
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
{
p0->current_data -= rw_len0;
p0->current_length += rw_len0;
-
tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
next0 = adj0[0].rewrite_header.next_index;
-
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
vnet_feature_arc_start_w_cfg_index
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
&error0);
-
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
ip6_hop_by_hop_option_t *opt0, *limit0;
ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
u8 type0;
-
s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
hbh0->protocol, (hbh0->length + 1) << 3, total_len);
}
else
{
- s =
- format (s, "\n unrecognized option %d length %d", type0,
- opt0->length);
+ s = format (s, "\n unrecognized option %d length %d", type0,
+ opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
}
else
{
- s =
- format (s, "\n unrecognized option %d length %d", type0,
- opt0->length);
+ s = format (s, "\n unrecognized option %d length %d", type0,
+ opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
#ifndef CLIB_MARCH_VARIANT
int
-vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
- u32 table_index)
+vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_interface_main_t *im = &vnm->interface_main;
fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
sw_if_index);
-
-
if (table_index != (u32) ~ 0)
{
dpo_id_t dpo = DPO_INVALID;
-
dpo_set (&dpo,
DPO_CLASSIFY,
DPO_PROTO_IP6,
classify_dpo_create (DPO_PROTO_IP6, table_index));
-
fib_table_entry_special_dpo_add (fib_index,
&pfx,
FIB_SOURCE_CLASSIFY,
to_ip4->checksum = ip4_header_checksum (to_ip4);
/* we've just done the IP checksum .. */
- to_b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (to_b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
rem -= len;
fo += len;
next_node_index = im->ah4_encrypt_node_index;
vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
- if (is_ipv6)
+ if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
- {
- tcp0->checksum =
- ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
- &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- }
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
- {
- udp0->checksum =
- ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
- &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
- }
- }
- else
- {
- if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
- {
- ip0->checksum = ip4_header_checksum (ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
- }
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
+ u32 oflags = vnet_buffer2 (b0)->oflags;
+
+ /*
+ * Clearing offload flags before checksum is computed
+ * It guarantees the cache hit!
+ */
+ vnet_buffer_offload_flags_clear (b0, oflags);
+
+ if (is_ipv6)
{
- tcp0->checksum =
- ip4_tcp_udp_compute_checksum (vm, b0, ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+ {
+ tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+ vm, b0, ip6_0, &bogus);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+ {
+ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+ vm, b0, ip6_0, &bogus);
+ }
}
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else
{
- udp0->checksum =
- ip4_tcp_udp_compute_checksum (vm, b0, ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
+ {
+ ip0->checksum = ip4_header_checksum (ip0);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+ {
+ tcp0->checksum =
+ ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+ {
+ udp0->checksum =
+ ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+ }
}
}
vlib_buffer_advance (b0, iph_offset);
else if (unformat (input, "buffer-flags %U",
unformat_vnet_buffer_flags, &s.buffer_flags))
;
-
+ else if (unformat (input, "buffer-offload-flags %U",
+ unformat_vnet_buffer_offload_flags, &s.buffer_oflags))
+ ;
else if (unformat (input, "node %U",
unformat_vlib_node, vm, &s.node_index))
;
}
static_always_inline void
-fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
- int gso_enabled, u32 gso_size)
+fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
+ u32 buffer_oflags, int gso_enabled, u32 gso_size)
{
for (int i = 0; i < n_buffers; i++)
{
vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
u8 l4_proto = 0;
+ u32 oflags = 0;
ethernet_header_t *eh =
(ethernet_header_t *) vlib_buffer_get_current (b0);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
l4_proto = ip4->protocol;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
if (l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
/* only set GSO flag for chained buffers */
if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
vnet_buffer (b)->feature_arc_index = feature_arc_index;
}
- if (pi->gso_enabled ||
- (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
+ if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD))
{
fill_buffer_offload_flags (vm, to_next, n_this_frame,
- pi->gso_enabled, pi->gso_size);
+ s->buffer_oflags, pi->gso_enabled,
+ pi->gso_size);
}
n_trace = vlib_get_trace_count (vm, node);
for max_packet_bytes. */
u32 buffer_bytes;
- /* Buffer flags to set in each packet e.g. checksum offload flags */
+ /* Buffer flags to set in each packet e.g. l2 valid flags */
u32 buffer_flags;
+ /* Buffer offload flags to set in each packet e.g. checksum offload flags */
+ u32 buffer_oflags;
+
/* Last packet length if packet size edit type is increment. */
u32 last_increment_packet_size;
}
else
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
return checksum;
}
uh->checksum = 0;
uh->length = clib_host_to_net_u16 (udp_len);
if (offload_csum)
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
return uh;
u8 const underlay_hdr_len = is_ip4 ?
sizeof (ip4_vxlan_gbp_header_t) : sizeof (ip6_vxlan_gbp_header_t);
u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
- u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
- VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
- u32 const inner_packet_csum_offload_flags =
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ u32 const csum_flags =
+ is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+ VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+ u32 const outer_packet_csum_offload_flags =
+ is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 const inner_packet_removed_flags =
VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
n_left_from -= 2;
u32 or_flags = b[0]->flags | b[1]->flags;
- if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
b[0]->flags |= csum_flags;
vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+ vnet_buffer_offload_flags_set (b[0],
+ outer_packet_csum_offload_flags);
b[1]->flags |= csum_flags;
vnet_buffer (b[1])->l3_hdr_offset = l3_1 - b[1]->data;
vnet_buffer (b[1])->l4_hdr_offset = (u8 *) udp1 - b[1]->data;
+ vnet_buffer_offload_flags_set (b[1],
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
n_left_from -= 1;
n_left_to_next -= 1;
- if (csum_offload && (b[0]->flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (b[0]->flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
b[0]->flags |= csum_flags;
vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+ vnet_buffer_offload_flags_set (b[0],
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
u8 const underlay_hdr_len = is_ip4 ?
sizeof (ip4_vxlan_header_t) : sizeof (ip6_vxlan_header_t);
u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
- u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
- VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
- u32 const inner_packet_csum_offload_flags =
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ u32 const csum_flags =
+ is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+ VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+ u32 const outer_packet_csum_offload_flags =
+ is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 const inner_packet_removed_flags =
VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
b += 2;
u32 or_flags = b0->flags | b1->flags;
- if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
b0->flags |= csum_flags;
vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+ vnet_buffer_offload_flags_set (b0,
+ outer_packet_csum_offload_flags);
b1->flags |= csum_flags;
vnet_buffer (b1)->l3_hdr_offset = l3_1 - b1->data;
vnet_buffer (b1)->l4_hdr_offset = (u8 *) udp1 - b1->data;
+ vnet_buffer_offload_flags_set (b1,
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
vlib_buffer_t *b0 = b[0];
b += 1;
- if (csum_offload && (b0->flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
b0->flags |= csum_flags;
vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+ vnet_buffer_offload_flags_set (b0,
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
" interface loop0\n"
" tx-interface loop1\n"
" node loop1-output\n"
- " buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n"
+ " buffer-flags ip4 offload\n"
+ " buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
" data {\n"
" IP4: 1.2.3 -> dead.0000.0001\n"
" UDP: 11.22.33.44 -> 11.22.34.44\n"
" interface loop0\n"
" tx-interface loop1\n"
" node loop1-output\n"
- " buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n"
+ " buffer-flags ip4 offload\n"
+ " buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
" data {\n"
" IP4: 1.2.3 -> dead.0000.0001\n"
" UDP: 11.22.33.44 -> 11.22.34.44\n"
Code Review / vpp.git / commitdiff