X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fdevices%2Fvirtio%2Fvhost_user_input.c;h=a371e57e8e38b7e60a10f8efede3bf1b1656b475;hb=b232d192f2f77114e26cf7f99b57bcd6140872d0;hp=286fa7ed744117a4e65fe87854aba358c7bd17d0;hpb=47727c02df4c79c21c6ceffaff65222c32cf6996;p=vpp.git diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 286fa7ed744..a371e57e8e3 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -39,6 +39,7 @@ #include #include +#include #include #include @@ -200,7 +201,7 @@ vhost_user_rx_discard_packet (vlib_main_t * vm, u16 last_used_idx = txvq->last_used_idx; while (discarded_packets != discard_max) { - if (avail_idx == txvq->last_avail_idx) + if (avail_idx == last_avail_idx) goto out; u16 desc_chain_head = txvq->avail->ring[last_avail_idx & mask]; @@ -243,12 +244,97 @@ vhost_user_input_rewind_buffers (vlib_main_t * vm, cpu->rx_buffers_len++; } +static_always_inline void +vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, + virtio_net_hdr_t * hdr) +{ + u8 l4_hdr_sz = 0; + + if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) + { + u8 l4_proto = 0; + ethernet_header_t *eh = (ethernet_header_t *) b0_data; + u16 ethertype = clib_net_to_host_u16 (eh->type); + u16 l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + vnet_buffer (b0)->l2_hdr_offset = 0; + vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz; + vnet_buffer (b0)->l4_hdr_offset = hdr->csum_start; + b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | + VNET_BUFFER_F_OFFLOAD_IP_CKSUM); + + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz); + l4_proto = ip4->protocol; + b0->flags |= VNET_BUFFER_F_IS_IP4; + } + else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip6 = (ip6_header_t *) (b0_data + l2hdr_sz); + l4_proto = ip6->protocol; + b0->flags |= VNET_BUFFER_F_IS_IP6; + } + + if (l4_proto == IP_PROTOCOL_TCP) + { + tcp_header_t *tcp = (tcp_header_t *) + (b0_data + vnet_buffer (b0)->l4_hdr_offset); + l4_hdr_sz = tcp_header_bytes (tcp); + tcp->checksum = 0; + b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + } + else if (l4_proto == IP_PROTOCOL_UDP) + { + udp_header_t *udp = + (udp_header_t *) (b0_data + vnet_buffer (b0)->l4_hdr_offset); + l4_hdr_sz = sizeof (*udp); + udp->checksum = 0; + b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + } + } + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP) + { + vnet_buffer2 (b0)->gso_size = hdr->gso_size; + vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; + b0->flags |= VNET_BUFFER_F_GSO; + } + else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4) + { + vnet_buffer2 (b0)->gso_size = hdr->gso_size; + vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; + b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4); + } + else if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6) + { + vnet_buffer2 (b0)->gso_size = hdr->gso_size; + vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; + b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6); + } +} + static_always_inline u32 vhost_user_if_input (vlib_main_t * vm, vhost_user_main_t * vum, vhost_user_intf_t * vui, u16 qid, vlib_node_runtime_t * node, - vnet_hw_interface_rx_mode mode) + vnet_hw_interface_rx_mode mode, u8 enable_csum) { vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; vnet_feature_main_t *fm = &feature_main; @@ -340,7 +426,7 @@ vhost_user_if_input (vlib_main_t * vm, /* * For small packets (<2kB), we will not need more than one vlib buffer - * per packet. In case packets are bigger, we will just yeld at some point + * per packet. In case packets are bigger, we will just yield at some point * in the loop and come back later. This is not an issue as for big packet, * processing cost really comes from the memory copy. * The assumption is that big packets will fit in 40 buffers. @@ -397,12 +483,13 @@ vhost_user_if_input (vlib_main_t * vm, vlib_frame_t *f; ethernet_input_frame_t *ef; nf = vlib_node_runtime_get_next_frame (vm, node, next_index); - f = vlib_get_frame (vm, nf->frame_index); + f = vlib_get_frame (vm, nf->frame); f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; ef = vlib_frame_scalar_args (f); ef->sw_if_index = vui->sw_if_index; ef->hw_if_index = vui->hw_if_index; + vlib_frame_no_append (f); } while (n_left > 0) @@ -445,7 +532,6 @@ vhost_user_if_input (vlib_main_t * vm, if (PREDICT_FALSE (n_trace)) { - //TODO: next_index is not exactly known at that point vlib_trace_buffer (vm, node, next_index, b_head, /* follow_chain */ 0); vhost_trace_t *t0 = @@ -471,16 +557,42 @@ vhost_user_if_input (vlib_main_t * vm, } } - if (PREDICT_TRUE (vui->is_any_layout) || - (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))) - { - /* ANYLAYOUT or single buffer */ - desc_data_offset = vui->virtio_net_hdr_sz; - } - else + desc_data_offset = vui->virtio_net_hdr_sz; + + if (enable_csum) { - /* CSR case without ANYLAYOUT, skip 1st buffer */ - desc_data_offset = desc_table[desc_current].len; + virtio_net_hdr_mrg_rxbuf_t *hdr; + u8 *b_data; + u16 current; + + hdr = map_guest_mem (vui, desc_table[desc_current].addr, &map_hint); + if (PREDICT_FALSE (hdr == 0)) + { + vlib_error_count (vm, node->node_index, + VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); + goto out; + } + if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) + { + if ((desc_data_offset == desc_table[desc_current].len) && + (desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT)) + { + current = desc_table[desc_current].next; + b_data = map_guest_mem (vui, desc_table[current].addr, + &map_hint); + if (PREDICT_FALSE (b_data == 0)) + { + vlib_error_count (vm, node->node_index, + VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, + 1); + goto out; + } + } + else + b_data = (u8 *) hdr + desc_data_offset; + + vhost_user_handle_rx_offload (b_head, b_data, &hdr->hdr); + } } while (1) @@ -531,6 +643,7 @@ vhost_user_if_input (vlib_main_t * vm, } /* Prepare a copy order executed later for the data */ + ASSERT (copy_len < VHOST_USER_COPY_ARRAY_N); vhost_copy_t *cpy = &cpu->copy[copy_len]; copy_len++; u32 desc_data_l = desc_table[desc_current].len - desc_data_offset; @@ -652,8 +765,14 @@ VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm, { vui = pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance); - n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, node, - dq->mode); + if (vui->features & (1ULL << FEAT_VIRTIO_NET_F_CSUM)) + n_rx_packets += + vhost_user_if_input (vm, vum, vui, dq->queue_id, node, dq->mode, + 1); + else + n_rx_packets += + vhost_user_if_input (vm, vum, vui, dq->queue_id, node, dq->mode, + 0); } } @@ -665,6 +784,7 @@ VLIB_REGISTER_NODE (vhost_user_input_node) = { .type = VLIB_NODE_TYPE_INPUT, .name = "vhost-user-input", .sibling_of = "device-input", + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, /* Will be enabled if/when hardware is detected. */ .state = VLIB_NODE_STATE_DISABLED,