X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fgso%2Fnode.c;h=b57830f1ac53030263ab4be395e69c1b4603755d;hb=84f91fa9c54f82c54b58ea3bf6e9ba22ff735d3a;hp=663f9cc602d0754d05efa152e8be7f84536ef133;hpb=5ec6a4ea4440211e468a813ffe0f3173421c9b29;p=vpp.git diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c index 663f9cc602d..b57830f1ac5 100644 --- a/src/vnet/gso/node.c +++ b/src/vnet/gso/node.c @@ -56,6 +56,50 @@ format_gso_trace (u8 * s, va_list * args) return s; } +static_always_inline u16 +tso_segment_ipip_tunnel_fixup (vlib_main_t * vm, + vnet_interface_per_thread_data_t * ptd, + vlib_buffer_t * sb0, + generic_header_offset_t * gho) +{ + u16 n_tx_bufs = vec_len (ptd->split_buffers); + u16 i = 0, n_tx_bytes = 0; + + while (i < n_tx_bufs) + { + vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); + vnet_get_outer_header (b0, gho); + clib_memcpy_fast (vlib_buffer_get_current (b0), + vlib_buffer_get_current (sb0), gho->outer_hdr_sz); + + ip4_header_t *ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + + gho->outer_l3_hdr_offset); + ip6_header_t *ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + + gho->outer_l3_hdr_offset); + + if (gho->gho_flags & GHO_F_OUTER_IP4) + { + ip4->length = + clib_host_to_net_u16 (b0->current_length - + gho->outer_l3_hdr_offset); + ip4->checksum = ip4_header_checksum (ip4); + } + else if (gho->gho_flags & GHO_F_OUTER_IP6) + { + ip6->payload_length = + clib_host_to_net_u16 (b0->current_length - + gho->outer_l4_hdr_offset); + } + + n_tx_bytes += gho->outer_hdr_sz; + i++; + } + return n_tx_bytes; + +} + static_always_inline void tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b, generic_header_offset_t * gho) @@ -177,14 +221,22 @@ static_always_inline void tso_init_buf_from_template_base (vlib_buffer_t * nb0, vlib_buffer_t * b0, u32 flags, u16 length) { + /* copying objects from cacheline 0 */ nb0->current_data = b0->current_data; - nb0->total_length_not_including_first_buffer = 0; + nb0->current_length = length; nb0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags; - nb0->trace_handle = b0->trace_handle; + nb0->flow_id = b0->flow_id; + nb0->error = b0->error; + nb0->current_config_index = b0->current_config_index; clib_memcpy_fast (&nb0->opaque, &b0->opaque, sizeof (nb0->opaque)); + + /* copying objects from cacheline 1 */ + nb0->trace_handle = b0->trace_handle; + nb0->total_length_not_including_first_buffer = 0; + + /* copying data */ clib_memcpy_fast (vlib_buffer_get_current (nb0), vlib_buffer_get_current (b0), length); - nb0->current_length = length; } static_always_inline void @@ -209,8 +261,7 @@ tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0, static_always_inline void tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, - int is_ip6, generic_header_offset_t * gho, - u32 do_csums) + int is_ip6, generic_header_offset_t * gho) { ip4_header_t *ip4 = (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset); @@ -224,9 +275,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, if (is_ip6) { ip6->payload_length = - clib_host_to_net_u16 (b0->current_length - - (gho->l4_hdr_offset - gho->l2_hdr_offset)); - if (do_csums && (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)) + clib_host_to_net_u16 (b0->current_length - gho->l4_hdr_offset); + if (gho->gho_flags & GHO_F_TCP) { int bogus = 0; tcp->checksum = 0; @@ -238,20 +288,29 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, else { ip4->length = - clib_host_to_net_u16 (b0->current_length - - (gho->l3_hdr_offset - gho->l2_hdr_offset)); - if (do_csums) + clib_host_to_net_u16 (b0->current_length - gho->l3_hdr_offset); + if (gho->gho_flags & GHO_F_IP4) + ip4->checksum = ip4_header_checksum (ip4); + if (gho->gho_flags & GHO_F_TCP) { - if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) - ip4->checksum = ip4_header_checksum (ip4); - if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) - { - tcp->checksum = 0; - tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4); - } - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + tcp->checksum = 0; + tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4); } + b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + } + + if ((gho->gho_flags & GHO_F_TUNNEL) == 0) + { + u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; + + ip_adjacency_t *adj0 = adj_get (adj_index0); + + if (adj0->lookup_next_index == IP_LOOKUP_NEXT_MIDCHAIN && + adj0->sub_type.midchain.fixup_func) + /* calls e.g. ipip44_fixup */ + adj0->sub_type.midchain.fixup_func + (vm, adj0, b0, adj0->sub_type.midchain.fixup_data); } } @@ -267,8 +326,7 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, static_always_inline u32 tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, u32 sbi0, vlib_buffer_t * sb0, - generic_header_offset_t * gho, u32 n_bytes_b0, int is_ip6, - u32 do_csums) + generic_header_offset_t * gho, u32 n_bytes_b0, int is_ip6) { u32 n_tx_bytes = 0; u16 gso_size = vnet_buffer2 (sb0)->gso_size; @@ -316,8 +374,7 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, src_ptr = vlib_buffer_get_current (sb0) + l234_sz + first_data_size; src_left = sb0->current_length - l234_sz - first_data_size; - tso_fixup_segmented_buf (vm, b0, tcp_flags_no_fin_psh, is_ip6, gho, - do_csums); + tso_fixup_segmented_buf (vm, b0, tcp_flags_no_fin_psh, is_ip6, gho); /* grab a second buffer and prepare the loop */ ASSERT (dbi < vec_len (ptd->split_buffers)); @@ -367,7 +424,7 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, { n_tx_bytes += cdb0->current_length; tso_fixup_segmented_buf (vm, cdb0, tcp_flags_no_fin_psh, is_ip6, - gho, do_csums); + gho); ASSERT (dbi < vec_len (ptd->split_buffers)); cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]); tso_init_buf_from_template (vm, cdb0, b0, l234_sz, @@ -376,8 +433,7 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, } } - tso_fixup_segmented_buf (vm, cdb0, save_tcp_flags, is_ip6, gho, - do_csums); + tso_fixup_segmented_buf (vm, cdb0, save_tcp_flags, is_ip6, gho); n_tx_bytes += cdb0->current_length; } @@ -411,7 +467,7 @@ vnet_gso_node_inline (vlib_main_t * vm, vlib_frame_t * frame, vnet_main_t * vnm, vnet_hw_interface_t * hi, - int is_ip6, int do_segmentation) + int is_l2, int is_ip4, int is_ip6, int do_segmentation) { u32 *to_next; u32 next_index = node->cached_next_index; @@ -496,7 +552,8 @@ vnet_gso_node_inline (vlib_main_t * vm, t0->flags = b[0]->flags & VNET_BUFFER_F_GSO; t0->gso_size = vnet_buffer2 (b[0])->gso_size; t0->gso_l4_hdr_sz = vnet_buffer2 (b[0])->gso_l4_hdr_sz; - vnet_generic_header_offset_parser (b[0], &t0->gho); + vnet_generic_header_offset_parser (b[0], &t0->gho, is_l2, + is_ip4, is_ip6); } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) { @@ -504,7 +561,8 @@ vnet_gso_node_inline (vlib_main_t * vm, t1->flags = b[1]->flags & VNET_BUFFER_F_GSO; t1->gso_size = vnet_buffer2 (b[1])->gso_size; t1->gso_l4_hdr_sz = vnet_buffer2 (b[1])->gso_l4_hdr_sz; - vnet_generic_header_offset_parser (b[1], &t1->gho); + vnet_generic_header_offset_parser (b[1], &t1->gho, is_l2, + is_ip4, is_ip6); } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) { @@ -512,7 +570,8 @@ vnet_gso_node_inline (vlib_main_t * vm, t2->flags = b[2]->flags & VNET_BUFFER_F_GSO; t2->gso_size = vnet_buffer2 (b[2])->gso_size; t2->gso_l4_hdr_sz = vnet_buffer2 (b[2])->gso_l4_hdr_sz; - vnet_generic_header_offset_parser (b[2], &t2->gho); + vnet_generic_header_offset_parser (b[2], &t2->gho, is_l2, + is_ip4, is_ip6); } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) { @@ -520,7 +579,8 @@ vnet_gso_node_inline (vlib_main_t * vm, t3->flags = b[3]->flags & VNET_BUFFER_F_GSO; t3->gso_size = vnet_buffer2 (b[3])->gso_size; t3->gso_l4_hdr_sz = vnet_buffer2 (b[3])->gso_l4_hdr_sz; - vnet_generic_header_offset_parser (b[3], &t3->gho); + vnet_generic_header_offset_parser (b[3], &t3->gho, is_l2, + is_ip4, is_ip6); } from += 4; @@ -547,7 +607,6 @@ vnet_gso_node_inline (vlib_main_t * vm, vnet_hw_interface_t *hi0; u32 next0 = 0; u32 do_segmentation0 = 0; - u32 do_csums = 0; swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; if (PREDICT_FALSE (hi->sw_if_index != swif0)) @@ -573,7 +632,8 @@ vnet_gso_node_inline (vlib_main_t * vm, t0->flags = b[0]->flags & VNET_BUFFER_F_GSO; t0->gso_size = vnet_buffer2 (b[0])->gso_size; t0->gso_l4_hdr_sz = vnet_buffer2 (b[0])->gso_l4_hdr_sz; - vnet_generic_header_offset_parser (b[0], &t0->gho); + vnet_generic_header_offset_parser (b[0], &t0->gho, is_l2, + is_ip4, is_ip6); } if (do_segmentation0) @@ -592,12 +652,14 @@ vnet_gso_node_inline (vlib_main_t * vm, u32 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b[0]); u32 n_tx_bytes = 0; - vnet_generic_header_offset_parser (b[0], &gho); + vnet_generic_header_offset_parser (b[0], &gho, is_l2, + is_ip4, is_ip6); if (PREDICT_FALSE (gho.gho_flags & GHO_F_TUNNEL)) { if (PREDICT_FALSE - ((gho.gho_flags & GHO_F_VXLAN_TUNNEL) == 0)) + (gho.gho_flags & (GHO_F_GRE_TUNNEL | + GHO_F_GENEVE_TUNNEL))) { /* not supported yet */ drop_one_buffer_and_count (vm, vnm, node, from - 1, @@ -610,17 +672,12 @@ vnet_gso_node_inline (vlib_main_t * vm, vnet_get_inner_header (b[0], &gho); n_bytes_b0 -= gho.outer_hdr_sz; - /* - * In case of tunnel encapsulated packet, we will - * calculate the checksums for segmented inner packets. - */ - do_csums = 1; is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0; } n_tx_bytes = tso_segment_buffer (vm, ptd, bi0, b[0], &gho, n_bytes_b0, - is_ip6, do_csums); + is_ip6); if (PREDICT_FALSE (n_tx_bytes == 0)) { @@ -631,12 +688,22 @@ vnet_gso_node_inline (vlib_main_t * vm, continue; } + if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL)) { vnet_get_outer_header (b[0], &gho); n_tx_bytes += tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho); } + else + if (PREDICT_FALSE + (gho.gho_flags & (GHO_F_IPIP_TUNNEL | + GHO_F_IPIP6_TUNNEL))) + { + vnet_get_outer_header (b[0], &gho); + n_tx_bytes += + tso_segment_ipip_tunnel_fixup (vm, ptd, b[0], &gho); + } u16 n_tx_bufs = vec_len (ptd->split_buffers); u32 *from_seg = ptd->split_buffers; @@ -645,30 +712,7 @@ vnet_gso_node_inline (vlib_main_t * vm, { u32 sbi0; vlib_buffer_t *sb0; - if (n_tx_bufs >= n_left_to_next) - { - while (n_left_to_next > 0) - { - sbi0 = to_next[0] = from_seg[0]; - sb0 = vlib_get_buffer (vm, sbi0); - ASSERT (sb0->current_length > 0); - to_next += 1; - from_seg += 1; - n_left_to_next -= 1; - n_tx_bufs -= 1; - vnet_feature_next (&next0, sb0); - vlib_validate_buffer_enqueue_x1 (vm, node, - next_index, - to_next, - n_left_to_next, - sbi0, next0); - } - vlib_put_next_frame (vm, node, next_index, - n_left_to_next); - vlib_get_new_next_frame (vm, node, next_index, - to_next, n_left_to_next); - } - while (n_tx_bufs > 0) + while (n_tx_bufs > 0 && n_left_to_next > 0) { sbi0 = to_next[0] = from_seg[0]; sb0 = vlib_get_buffer (vm, sbi0); @@ -677,6 +721,7 @@ vnet_gso_node_inline (vlib_main_t * vm, from_seg += 1; n_left_to_next -= 1; n_tx_bufs -= 1; + next0 = 0; vnet_feature_next (&next0, sb0); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -684,6 +729,11 @@ vnet_gso_node_inline (vlib_main_t * vm, n_left_to_next, sbi0, next0); } + vlib_put_next_frame (vm, node, next_index, + n_left_to_next); + if (n_tx_bufs > 0) + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); } /* The buffers were enqueued. Reset the length */ _vec_len (ptd->split_buffers) = 0; @@ -707,7 +757,8 @@ vnet_gso_node_inline (vlib_main_t * vm, static_always_inline uword vnet_gso_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame, int is_ip6) + vlib_node_runtime_t * node, vlib_frame_t * frame, int is_l2, + int is_ip4, int is_ip6) { vnet_main_t *vnm = vnet_get_main (); vnet_hw_interface_t *hi; @@ -721,10 +772,12 @@ vnet_gso_inline (vlib_main_t * vm, if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) return vnet_gso_node_inline (vm, node, frame, vnm, hi, - is_ip6, /* do_segmentation */ 0); + is_l2, is_ip4, is_ip6, + /* do_segmentation */ 0); else return vnet_gso_node_inline (vm, node, frame, vnm, hi, - is_ip6, /* do_segmentation */ 1); + is_l2, is_ip4, is_ip6, + /* do_segmentation */ 1); } return 0; } @@ -732,25 +785,29 @@ vnet_gso_inline (vlib_main_t * vm, VLIB_NODE_FN (gso_l2_ip4_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return vnet_gso_inline (vm, node, frame, 0 /* ip6 */ ); + return vnet_gso_inline (vm, node, frame, 1 /* l2 */ , 1 /* ip4 */ , + 0 /* ip6 */ ); } VLIB_NODE_FN (gso_l2_ip6_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return vnet_gso_inline (vm, node, frame, 1 /* ip6 */ ); + return vnet_gso_inline (vm, node, frame, 1 /* l2 */ , 0 /* ip4 */ , + 1 /* ip6 */ ); } VLIB_NODE_FN (gso_ip4_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return vnet_gso_inline (vm, node, frame, 0 /* ip6 */ ); + return vnet_gso_inline (vm, node, frame, 0 /* l2 */ , 1 /* ip4 */ , + 0 /* ip6 */ ); } VLIB_NODE_FN (gso_ip6_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - return vnet_gso_inline (vm, node, frame, 1 /* ip6 */ ); + return vnet_gso_inline (vm, node, frame, 0 /* l2 */ , 0 /* ip4 */ , + 1 /* ip6 */ ); } /* *INDENT-OFF* */ @@ -806,15 +863,13 @@ VNET_FEATURE_INIT (gso_l2_ip6_node, static) = { VNET_FEATURE_INIT (gso_ip4_node, static) = { .arc_name = "ip4-output", .node_name = "gso-ip4", - .runs_after = VNET_FEATURES ("ipsec4-output-feature"), - .runs_before = VNET_FEATURES ("interface-output"), + .runs_before = VNET_FEATURES ("esp4-encrypt-tun","ipsec4-output-feature"), }; VNET_FEATURE_INIT (gso_ip6_node, static) = { .arc_name = "ip6-output", .node_name = "gso-ip6", - .runs_after = VNET_FEATURES ("ipsec6-output-feature"), - .runs_before = VNET_FEATURES ("interface-output"), + .runs_before = VNET_FEATURES ("esp6-encrypt-tun","ipsec6-output-feature"), }; /*